Skip to content

Commit

Permalink
Added initial implementation for Google Cloud Storage
Browse files Browse the repository at this point in the history
  • Loading branch information
amankrx committed Jan 9, 2025
1 parent b1df876 commit e465ce3
Show file tree
Hide file tree
Showing 8 changed files with 1,097 additions and 2 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ bazel-*
target/
.vscode/
.zed
.idea/
.cache
.terraform*
.config
Expand Down
53 changes: 53 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

95 changes: 94 additions & 1 deletion nativelink-config/src/stores.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,33 @@ pub enum StoreSpec {
/// "delay": 0.3,
/// "jitter": 0.5
/// },
/// "multipart_max_concurrent_uploads": 10
/// "max_concurrent_uploads": 10
/// }
/// ```
///
experimental_s3_store(S3Spec),

/// GCS store will use Google's GCS service as a backend to store
/// the files. This configuration can be used to share files
/// across multiple instances.
///
/// **Example JSON Config:**
/// ```json
/// "experimental_gcs_store": {
/// "project_id": "sample-project",
/// "bucket": "test-bucket",
/// "key_prefix": "test-prefix-index/",
/// "retry": {
/// "max_retries": 6,
/// "delay": 0.3,
/// "jitter": 0.5
/// },
/// "multipart_max_concurrent_uploads": 10
/// }
/// ```
///
experimental_gcs_store(GcsSpec),

/// Verify store is used to apply verifications to an underlying
/// store implementation. It is strongly encouraged to validate
/// as much data as you can before accepting data from a client,
Expand Down Expand Up @@ -787,6 +808,78 @@ pub struct S3Spec {
pub disable_http2: bool,
}

#[derive(Serialize, Deserialize, Debug, Default, Clone)]
#[serde(deny_unknown_fields)]
pub struct GcsSpec {
/// Project ID for the GCS service
#[serde(default, deserialize_with = "convert_string_with_shellexpand")]
pub project_id: String,

/// Bucket name to use as the backend
#[serde(default, deserialize_with = "convert_string_with_shellexpand")]
pub bucket: String,

/// If you wish to prefix the location in GCS. If None, no prefix will be used.
#[serde(default)]
pub key_prefix: Option<String>,

/// Retry configuration to use when a network request fails.
#[serde(default)]
pub retry: Retry,

/// If the number of seconds since the `last_modified` time of the object
/// is greater than this value, the object will not be considered
/// "existing". This allows for external tools to delete objects that
/// have not been uploaded in a long time. If a client receives a `NotFound`
/// the client should re-upload the object.
///
/// There should be sufficient buffer time between how long the expiration
/// configuration of the external tool is and this value. Keeping items
/// around for a few days is generally a good idea.
///
/// Default: 0. Zero means never consider an object expired.
#[serde(default, deserialize_with = "convert_duration_with_shellexpand")]
pub consider_expired_after_s: u32,

/// The maximum buffer size to retain in case of a retryable error
/// during upload. Setting this to zero will disable upload buffering;
/// this means that in the event of a failure during upload, the entire
/// upload will be aborted and the client will likely receive an error.
///
/// Default: 5MB.
pub max_retry_buffer_per_request: Option<usize>,

/// Size of chunks for resumeable uploads (in bytes).
/// Must be a multiple of 256 KB.
///
/// Default: 8MB.
pub resumable_chunk_size: Option<usize>,

/// Maximum number of concurrent uploads for resumable operations
///
/// Default: 10.
pub max_concurrent_uploads: Option<usize>,

/// Optional endpoint override for testing
/// Example: "localhost:8080" for local development
///
/// Default: None (uses production GCS endpoint)
#[serde(default)]
pub endpoint: Option<String>,

/// Allow unencrypted HTTP connections. Only use this for local testing.
///
/// Default: false
#[serde(default)]
pub insecure_allow_http: bool,

/// Disable http/2 connections and only use http/1.1.
///
/// Default: false
#[serde(default)]
pub disable_http2: bool,
}

#[allow(non_camel_case_types)]
#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
pub enum StoreType {
Expand Down
4 changes: 4 additions & 0 deletions nativelink-store/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ rust_library(
"src/size_partitioning_store.rs",
"src/store_manager.rs",
"src/verify_store.rs",
"src/gcs_store.rs",
],
proc_macro_deps = [
"@crates//:async-trait",
Expand All @@ -52,9 +53,11 @@ rust_library(
"@crates//:bytes",
"@crates//:bytes-utils",
"@crates//:const_format",
"@crates//:crc32c",
"@crates//:filetime",
"@crates//:fred",
"@crates//:futures",
"@crates//:googleapis-tonic-google-storage-v2",
"@crates//:hex",
"@crates//:http-body",
"@crates//:hyper-0.14.31",
Expand All @@ -70,6 +73,7 @@ rust_library(
"@crates//:tokio-util",
"@crates//:tonic",
"@crates//:tracing",
"@crates//:urlencoding",
"@crates//:uuid",
],
)
Expand Down
5 changes: 4 additions & 1 deletion nativelink-store/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ byteorder = { version = "1.5.0", default-features = false }
bytes = { version = "1.9.0", default-features = false }
bytes-utils = { version = "0.1.4", default-features = false }
const_format = { version = "0.2.34", default-features = false }
crc32c = "0.6.8"
filetime = "0.2.25"
fred = { version = "10.0.1", default-features = false, features = [
"i-std",
Expand All @@ -38,6 +39,7 @@ fred = { version = "10.0.1", default-features = false, features = [
"sentinel-auth",
"subscriber-client",
] }
googleapis-tonic-google-storage-v2 = "0.17.0"
patricia_tree = { version = "0.8.0", default-features = false }
futures = { version = "0.3.31", default-features = false }
hex = { version = "0.4.3", default-features = false }
Expand All @@ -54,9 +56,10 @@ serde = { version = "1.0.216", default-features = false }
tokio = { version = "1.42.0", features = ["fs", "rt-multi-thread", "signal", "io-util"], default-features = false }
tokio-stream = { version = "0.1.17", features = ["fs"], default-features = false }
tokio-util = { version = "0.7.13" }
tonic = { version = "0.12.3", features = ["transport", "tls"], default-features = false }
tonic = { version = "0.12.3", features = ["transport", "tls", "tls-native-roots"], default-features = false }
tracing = { version = "0.1.41", default-features = false }
uuid = { version = "1.11.0", default-features = false, features = ["v4", "serde"] }
urlencoding = "2.1.3"

[dev-dependencies]
nativelink-macro = { path = "../nativelink-macro" }
Expand Down
2 changes: 2 additions & 0 deletions nativelink-store/src/default_store_factory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ use crate::dedup_store::DedupStore;
use crate::existence_cache_store::ExistenceCacheStore;
use crate::fast_slow_store::FastSlowStore;
use crate::filesystem_store::FilesystemStore;
use crate::gcs_store::GcsStore;
use crate::grpc_store::GrpcStore;
use crate::memory_store::MemoryStore;
use crate::noop_store::NoopStore;
Expand All @@ -51,6 +52,7 @@ pub fn store_factory<'a>(
let store: Arc<dyn StoreDriver> = match backend {
StoreSpec::memory(spec) => MemoryStore::new(spec),
StoreSpec::experimental_s3_store(spec) => S3Store::new(spec, SystemTime::now).await?,
StoreSpec::experimental_gcs_store(spec) => GcsStore::new(spec, SystemTime::now).await?,
StoreSpec::redis_store(spec) => RedisStore::new(spec.clone())?,
StoreSpec::verify(spec) => VerifyStore::new(
spec,
Expand Down
Loading

0 comments on commit e465ce3

Please sign in to comment.