Make stores and schedulers lists of named specs

To migrate, change your config like this (similar for schedulers): ``` // Old: "stores": { "SOMESTORE": { "memory": {} } } // New: "stores": [ { "name": "SOMESTORE", "memory": {} } ] ``` For now this is not a breaking change, but a backwards compatible one. The backwards-compatibility code will be removed after the next release in a breaking change. Closes TraceMachina#834
aaronmondal · Nov 25, 2024 · d80dd7b · d80dd7b
1 parent afe0f4c
commit d80dd7b
Show file tree

Hide file tree

Showing 18 changed files with 312 additions and 94 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/deployment-examples/docker-compose/local-storage-cas.json b/deployment-examples/docker-compose/local-storage-cas.json
@@ -4,8 +4,9 @@
 // so objects are compressed, deduplicated and uses some in-memory
 // optimizations for certain hot paths.
 {
-  "stores": {
-    "CAS_MAIN_STORE": {
+  "stores": [
+    {
+      "name": "CAS_MAIN_STORE",
       "compression": {
         "compression_algorithm": {
           "lz4": {}
@@ -21,8 +22,8 @@
           }
         }
       }
-    },
-    "AC_MAIN_STORE": {
+    }, {
+      "name": "AC_MAIN_STORE",
       "filesystem": {
         "content_path": "~/.cache/nativelink/content_path-ac",
         "temp_path": "~/.cache/nativelink/tmp_path-ac",
@@ -32,7 +33,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {

diff --git a/deployment-examples/docker-compose/scheduler.json b/deployment-examples/docker-compose/scheduler.json
@@ -1,6 +1,7 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
+      "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "main",
@@ -9,8 +10,8 @@
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
+      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "main",
@@ -20,9 +21,10 @@
         "store_type": "ac"
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
+  ],
+  "schedulers": [
+    {
+      "name": "MAIN_SCHEDULER",
       "simple": {
         "supported_platform_properties": {
           "cpu_count": "minimum",
@@ -31,7 +33,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {

diff --git a/deployment-examples/docker-compose/worker.json b/deployment-examples/docker-compose/worker.json
@@ -1,6 +1,7 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
+      "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "main",
@@ -9,8 +10,8 @@
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
+      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "main",
@@ -19,8 +20,8 @@
         ],
         "store_type": "ac"
       }
-    },
-    "WORKER_FAST_SLOW_STORE": {
+    }, {
+      "name": "WORKER_FAST_SLOW_STORE",
       "fast_slow": {
         "fast": {
           "filesystem": {
@@ -39,7 +40,7 @@
         }
       }
     }
-  },
+  ],
   "workers": [{
     "local": {
       "worker_api_endpoint": {

diff --git a/flake.nix b/flake.nix
@@ -502,6 +502,7 @@
               pkgs.cosign
               pkgs.kubectl
               pkgs.kubernetes-helm
+              pkgs.kubectx
               pkgs.cilium-cli
               pkgs.vale
               pkgs.trivy

diff --git a/kubernetes/configmaps/cas.json b/kubernetes/configmaps/cas.json
@@ -2,8 +2,9 @@
 // `~/.cache/nativelink`. When this location is mounted as a PersistentVolume
 // it persists the cache across restarts.
 {
-  "stores": {
-    "CAS_MAIN_STORE": {
+  "stores": [
+    {
+      "name": "CAS_MAIN_STORE",
       "existence_cache": {
         "backend": {
           "compression": {
@@ -23,8 +24,8 @@
           }
         }
       }
-    },
-    "AC_MAIN_STORE": {
+    }, {
+      "name": "AC_MAIN_STORE",
       "completeness_checking": {
         "backend": {
           "filesystem": {
@@ -43,7 +44,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {

diff --git a/kubernetes/configmaps/scheduler.json b/kubernetes/configmaps/scheduler.json
@@ -1,6 +1,7 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
+      "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "main",
@@ -9,8 +10,8 @@
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
+      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "main",
@@ -20,9 +21,10 @@
         "store_type": "ac"
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
+  ],
+  "schedulers": [
+    {
+      "name": "MAIN_SCHEDULER",
       // TODO(adams): use the right scheduler because reclient doesn't use the cached results?
       // TODO(adams): max_bytes_per_stream
       "simple": {
@@ -46,7 +48,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {

diff --git a/kubernetes/configmaps/worker.json b/kubernetes/configmaps/worker.json
@@ -1,6 +1,7 @@
 {
-  "stores": {
-    "GRPC_LOCAL_STORE": {
+  "stores": [
+    {
+      "name": "GRPC_LOCAL_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "main",
@@ -9,8 +10,8 @@
         ],
         "store_type": "cas"
       }
-    },
-    "GRPC_LOCAL_AC_STORE": {
+    }, {
+      "name": "GRPC_LOCAL_AC_STORE",
       // Note: This file is used to test GRPC store.
       "grpc": {
         "instance_name": "main",
@@ -19,8 +20,8 @@
         ],
         "store_type": "ac"
       }
-    },
-    "WORKER_FAST_SLOW_STORE": {
+    }, {
+      "name": "WORKER_FAST_SLOW_STORE",
       "fast_slow": {
         "fast": {
           "filesystem": {
@@ -39,7 +40,7 @@
         }
       }
     }
-  },
+  ],
   "workers": [{
     "local": {
       "worker_api_endpoint": {

diff --git a/nativelink-config/BUILD.bazel b/nativelink-config/BUILD.bazel
@@ -31,6 +31,7 @@ rust_test_suite(
     name = "integration",
     timeout = "short",
     srcs = [
+        "tests/backwards_compat_test.rs",
         "tests/deserialization_test.rs",
     ],
     deps = [
@@ -40,6 +41,7 @@ rust_test_suite(
         "@crates//:humantime",
         "@crates//:pretty_assertions",
         "@crates//:serde",
+        "@crates//:serde_json",
         "@crates//:serde_json5",
     ],
 )

diff --git a/nativelink-config/Cargo.toml b/nativelink-config/Cargo.toml
@@ -12,3 +12,4 @@ shellexpand = { version = "3.1.0", default-features = false, features = ["base-0
 
 [dev-dependencies]
 pretty_assertions = { version = "1.4.1", features = ["std"] }
+serde_json = "1.0.133"
diff --git a/nativelink-config/examples/basic_cas.json b/nativelink-config/examples/basic_cas.json
@@ -1,6 +1,7 @@
 {
-  "stores": {
-    "AC_MAIN_STORE": {
+  "stores": [
+    {
+      "name": "AC_MAIN_STORE",
       "filesystem": {
         "content_path": "/tmp/nativelink/data-worker-test/content_path-ac",
         "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac",
@@ -9,8 +10,8 @@
           "max_bytes": 1000000000,
         }
       }
-    },
-    "WORKER_FAST_SLOW_STORE": {
+    }, {
+      "name": "WORKER_FAST_SLOW_STORE",
       "fast_slow": {
         // "fast" must be a "filesystem" store because the worker uses it to make
         // hardlinks on disk to a directory where the jobs are running.
@@ -34,9 +35,10 @@
         }
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
+  ],
+  "schedulers": [
+    {
+      "name": "MAIN_SCHEDULER",
       "simple": {
         "supported_platform_properties": {
           "cpu_count": "minimum",
@@ -61,7 +63,7 @@
         }
       }
     }
-  },
+  ],
   "workers": [{
     "local": {
       "worker_api_endpoint": {

diff --git a/nativelink-config/examples/filesystem_cas.json b/nativelink-config/examples/filesystem_cas.json
@@ -4,8 +4,9 @@
 // so objects are compressed, deduplicated and uses some in-memory
 // optimizations for certain hot paths.
 {
-  "stores": {
-    "FS_CONTENT_STORE": {
+  "stores": [
+    {
+      "name": "FS_CONTENT_STORE",
       "compression": {
         "compression_algorithm": {
           "lz4": {}
@@ -21,8 +22,8 @@
           }
         }
       }
-    },
-    "CAS_MAIN_STORE": {
+    }, {
+      "name": "CAS_MAIN_STORE",
       "verify": {
         "backend": {
           // Because we are using a dedup store, we can bypass small objects
@@ -78,8 +79,8 @@
         "verify_size": true,
         "verify_hash": true
       }
-    },
-    "AC_MAIN_STORE": {
+    }, {
+      "name": "AC_MAIN_STORE",
       "filesystem": {
         "content_path": "/tmp/nativelink/data/content_path-ac",
         "temp_path": "/tmp/nativelink/data/tmp_path-ac",
@@ -89,9 +90,10 @@
         }
       }
     }
-  },
-  "schedulers": {
-    "MAIN_SCHEDULER": {
+  ],
+  "schedulers": [
+    {
+      "name": "MAIN_SCHEDULER",
       "simple": {
         "supported_platform_properties": {
           "cpu_count": "minimum",
@@ -112,7 +114,7 @@
         }
       }
     }
-  },
+  ],
   "servers": [{
     "listener": {
       "http": {
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,3 +12,4 @@ shellexpand = { version = "3.1.0", default-features = false, features = ["base-0

		[dev-dependencies]
		pretty_assertions = { version = "1.4.1", features = ["std"] }
		serde_json = "1.0.133"