From c33d11190706c12148964847270756963de027b0 Mon Sep 17 00:00:00 2001 From: Aaron Siddhartha Mondal Date: Sun, 24 Nov 2024 01:39:45 +0100 Subject: [PATCH] [Breaking] Make stores and schedulers lists of named specs To migrate, change your config like this (similar for schedulers): ``` // Old: "stores": { "SOMESTORE": { "memory": {} } } // New: "stores": [ { "name": "SOMESTORE", "memory": {} } ] ``` Closes #834 --- .../docker-compose/local-storage-cas.json | 11 ++++---- .../docker-compose/scheduler.json | 18 ++++++------ .../docker-compose/worker.json | 15 +++++----- flake.nix | 1 + kubernetes/configmaps/cas.json | 11 ++++---- kubernetes/configmaps/scheduler.json | 18 ++++++------ kubernetes/configmaps/worker.json | 15 +++++----- nativelink-config/examples/basic_cas.json | 18 ++++++------ .../examples/filesystem_cas.json | 22 ++++++++------- nativelink-config/examples/redis.json | 19 +++++++------ .../s3_backend_with_local_fast_cas.json | 18 ++++++------ nativelink-config/src/cas_server.rs | 5 ++-- nativelink-config/src/lib.rs | 9 ++++++ src/bin/nativelink.rs | 28 +++++++++---------- 14 files changed, 117 insertions(+), 91 deletions(-) diff --git a/deployment-examples/docker-compose/local-storage-cas.json b/deployment-examples/docker-compose/local-storage-cas.json index e7b622043..09dd656cd 100644 --- a/deployment-examples/docker-compose/local-storage-cas.json +++ b/deployment-examples/docker-compose/local-storage-cas.json @@ -4,8 +4,9 @@ // so objects are compressed, deduplicated and uses some in-memory // optimizations for certain hot paths. { - "stores": { - "CAS_MAIN_STORE": { + "stores": [ + { + "name": "CAS_MAIN_STORE", "compression": { "compression_algorithm": { "lz4": {} @@ -21,8 +22,8 @@ } } } - }, - "AC_MAIN_STORE": { + }, { + "name": "AC_MAIN_STORE", "filesystem": { "content_path": "~/.cache/nativelink/content_path-ac", "temp_path": "~/.cache/nativelink/tmp_path-ac", @@ -32,7 +33,7 @@ } } } - }, + ], "servers": [{ "listener": { "http": { diff --git a/deployment-examples/docker-compose/scheduler.json b/deployment-examples/docker-compose/scheduler.json index 4af051738..e971ab2d9 100644 --- a/deployment-examples/docker-compose/scheduler.json +++ b/deployment-examples/docker-compose/scheduler.json @@ -1,6 +1,7 @@ { - "stores": { - "GRPC_LOCAL_STORE": { + "stores": [ + { + "name": "GRPC_LOCAL_STORE", // Note: This file is used to test GRPC store. "grpc": { "instance_name": "main", @@ -9,8 +10,8 @@ ], "store_type": "cas" } - }, - "GRPC_LOCAL_AC_STORE": { + }, { + "name": "GRPC_LOCAL_AC_STORE", // Note: This file is used to test GRPC store. "grpc": { "instance_name": "main", @@ -20,9 +21,10 @@ "store_type": "ac" } } - }, - "schedulers": { - "MAIN_SCHEDULER": { + ], + "schedulers": [ + { + "name": "MAIN_SCHEDULER", "simple": { "supported_platform_properties": { "cpu_count": "minimum", @@ -31,7 +33,7 @@ } } } - }, + ], "servers": [{ "listener": { "http": { diff --git a/deployment-examples/docker-compose/worker.json b/deployment-examples/docker-compose/worker.json index 186722e01..7e6d36fc7 100644 --- a/deployment-examples/docker-compose/worker.json +++ b/deployment-examples/docker-compose/worker.json @@ -1,6 +1,7 @@ { - "stores": { - "GRPC_LOCAL_STORE": { + "stores": [ + { + "name": "GRPC_LOCAL_STORE", // Note: This file is used to test GRPC store. "grpc": { "instance_name": "main", @@ -9,8 +10,8 @@ ], "store_type": "cas" } - }, - "GRPC_LOCAL_AC_STORE": { + }, { + "name": "GRPC_LOCAL_AC_STORE", // Note: This file is used to test GRPC store. "grpc": { "instance_name": "main", @@ -19,8 +20,8 @@ ], "store_type": "ac" } - }, - "WORKER_FAST_SLOW_STORE": { + }, { + "name": "WORKER_FAST_SLOW_STORE", "fast_slow": { "fast": { "filesystem": { @@ -39,7 +40,7 @@ } } } - }, + ], "workers": [{ "local": { "worker_api_endpoint": { diff --git a/flake.nix b/flake.nix index 4a0845e1d..98f0c5747 100644 --- a/flake.nix +++ b/flake.nix @@ -502,6 +502,7 @@ pkgs.cosign pkgs.kubectl pkgs.kubernetes-helm + pkgs.kubectx pkgs.cilium-cli pkgs.vale pkgs.trivy diff --git a/kubernetes/configmaps/cas.json b/kubernetes/configmaps/cas.json index e26b9c1cb..3487a7074 100644 --- a/kubernetes/configmaps/cas.json +++ b/kubernetes/configmaps/cas.json @@ -2,8 +2,9 @@ // `~/.cache/nativelink`. When this location is mounted as a PersistentVolume // it persists the cache across restarts. { - "stores": { - "CAS_MAIN_STORE": { + "stores": [ + { + "name": "CAS_MAIN_STORE", "existence_cache": { "backend": { "compression": { @@ -23,8 +24,8 @@ } } } - }, - "AC_MAIN_STORE": { + }, { + "name": "AC_MAIN_STORE", "completeness_checking": { "backend": { "filesystem": { @@ -43,7 +44,7 @@ } } } - }, + ], "servers": [{ "listener": { "http": { diff --git a/kubernetes/configmaps/scheduler.json b/kubernetes/configmaps/scheduler.json index e41060209..0341efb14 100644 --- a/kubernetes/configmaps/scheduler.json +++ b/kubernetes/configmaps/scheduler.json @@ -1,6 +1,7 @@ { - "stores": { - "GRPC_LOCAL_STORE": { + "stores": [ + { + "name": "GRPC_LOCAL_STORE", // Note: This file is used to test GRPC store. "grpc": { "instance_name": "main", @@ -9,8 +10,8 @@ ], "store_type": "cas" } - }, - "GRPC_LOCAL_AC_STORE": { + }, { + "name": "GRPC_LOCAL_AC_STORE", // Note: This file is used to test GRPC store. "grpc": { "instance_name": "main", @@ -20,9 +21,10 @@ "store_type": "ac" } } - }, - "schedulers": { - "MAIN_SCHEDULER": { + ], + "schedulers": [ + { + "name": "MAIN_SCHEDULER", // TODO(adams): use the right scheduler because reclient doesn't use the cached results? // TODO(adams): max_bytes_per_stream "simple": { @@ -46,7 +48,7 @@ } } } - }, + ], "servers": [{ "listener": { "http": { diff --git a/kubernetes/configmaps/worker.json b/kubernetes/configmaps/worker.json index 2a3d2911d..c5b8b1545 100644 --- a/kubernetes/configmaps/worker.json +++ b/kubernetes/configmaps/worker.json @@ -1,6 +1,7 @@ { - "stores": { - "GRPC_LOCAL_STORE": { + "stores": [ + { + "name": "GRPC_LOCAL_STORE", // Note: This file is used to test GRPC store. "grpc": { "instance_name": "main", @@ -9,8 +10,8 @@ ], "store_type": "cas" } - }, - "GRPC_LOCAL_AC_STORE": { + }, { + "name": "GRPC_LOCAL_AC_STORE", // Note: This file is used to test GRPC store. "grpc": { "instance_name": "main", @@ -19,8 +20,8 @@ ], "store_type": "ac" } - }, - "WORKER_FAST_SLOW_STORE": { + }, { + "name": "WORKER_FAST_SLOW_STORE", "fast_slow": { "fast": { "filesystem": { @@ -39,7 +40,7 @@ } } } - }, + ], "workers": [{ "local": { "worker_api_endpoint": { diff --git a/nativelink-config/examples/basic_cas.json b/nativelink-config/examples/basic_cas.json index 173951deb..ff8f03449 100644 --- a/nativelink-config/examples/basic_cas.json +++ b/nativelink-config/examples/basic_cas.json @@ -1,6 +1,7 @@ { - "stores": { - "AC_MAIN_STORE": { + "stores": [ + { + "name": "AC_MAIN_STORE", "filesystem": { "content_path": "/tmp/nativelink/data-worker-test/content_path-ac", "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac", @@ -9,8 +10,8 @@ "max_bytes": 1000000000, } } - }, - "WORKER_FAST_SLOW_STORE": { + }, { + "name": "WORKER_FAST_SLOW_STORE", "fast_slow": { // "fast" must be a "filesystem" store because the worker uses it to make // hardlinks on disk to a directory where the jobs are running. @@ -34,9 +35,10 @@ } } } - }, - "schedulers": { - "MAIN_SCHEDULER": { + ], + "schedulers": [ + { + "name": "MAIN_SCHEDULER", "simple": { "supported_platform_properties": { "cpu_count": "minimum", @@ -61,7 +63,7 @@ } } } - }, + ], "workers": [{ "local": { "worker_api_endpoint": { diff --git a/nativelink-config/examples/filesystem_cas.json b/nativelink-config/examples/filesystem_cas.json index 6f43bf6be..067f41fc2 100644 --- a/nativelink-config/examples/filesystem_cas.json +++ b/nativelink-config/examples/filesystem_cas.json @@ -4,8 +4,9 @@ // so objects are compressed, deduplicated and uses some in-memory // optimizations for certain hot paths. { - "stores": { - "FS_CONTENT_STORE": { + "stores": [ + { + "name": "FS_CONTENT_STORE", "compression": { "compression_algorithm": { "lz4": {} @@ -21,8 +22,8 @@ } } } - }, - "CAS_MAIN_STORE": { + }, { + "name": "CAS_MAIN_STORE", "verify": { "backend": { // Because we are using a dedup store, we can bypass small objects @@ -78,8 +79,8 @@ "verify_size": true, "verify_hash": true } - }, - "AC_MAIN_STORE": { + }, { + "name": "AC_MAIN_STORE", "filesystem": { "content_path": "/tmp/nativelink/data/content_path-ac", "temp_path": "/tmp/nativelink/data/tmp_path-ac", @@ -89,9 +90,10 @@ } } } - }, - "schedulers": { - "MAIN_SCHEDULER": { + ], + "schedulers": [ + { + "name": "MAIN_SCHEDULER", "simple": { "supported_platform_properties": { "cpu_count": "minimum", @@ -112,7 +114,7 @@ } } } - }, + ], "servers": [{ "listener": { "http": { diff --git a/nativelink-config/examples/redis.json b/nativelink-config/examples/redis.json index 1f06fe560..a990c28e9 100644 --- a/nativelink-config/examples/redis.json +++ b/nativelink-config/examples/redis.json @@ -1,18 +1,19 @@ { - "stores": { - "CAS_FAST_SLOW_STORE": { + "stores": [ + { + "name": "CAS_FAST_SLOW_STORE", "redis_store": { "addresses": ["redis://127.0.0.1:6379/"], "mode": "cluster" } - }, - "AC_FAST_SLOW_STORE": { + }, { + "name": "AC_FAST_SLOW_STORE", "redis_store": { "addresses": ["redis://127.0.0.1:6379/"], "mode": "cluster" } - }, - "AC_MAIN_STORE": { + }, { + "name": "AC_MAIN_STORE", "completeness_checking": { "backend": { "ref_store": { @@ -25,8 +26,8 @@ } } } - }, - "CAS_MAIN_STORE": { + }, { + "name": "CAS_MAIN_STORE", "existence_cache": { "backend": { "compression": { @@ -42,7 +43,7 @@ } } } - }, + ], "servers": [ { "listener": { diff --git a/nativelink-config/examples/s3_backend_with_local_fast_cas.json b/nativelink-config/examples/s3_backend_with_local_fast_cas.json index d74dd910d..031812969 100644 --- a/nativelink-config/examples/s3_backend_with_local_fast_cas.json +++ b/nativelink-config/examples/s3_backend_with_local_fast_cas.json @@ -1,6 +1,7 @@ { - "stores": { - "CAS_MAIN_STORE": { + "stores": [ + { + "name": "CAS_MAIN_STORE", "verify": { "backend": { "dedup": { @@ -70,8 +71,8 @@ "verify_size": true, "hash_verification_function": "sha256" } - }, - "AC_MAIN_STORE": { + }, { + "name": "AC_MAIN_STORE", "fast_slow": { "fast": { "memory": { @@ -104,9 +105,10 @@ } } } - }, - "schedulers": { - "MAIN_SCHEDULER": { + ], + "schedulers": [ + { + "name": "MAIN_SCHEDULER", "simple": { "supported_platform_properties": { "cpu_count": "minimum", @@ -127,7 +129,7 @@ } } } - }, + ], "servers": [{ "listener": { "http": { diff --git a/nativelink-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs index a258b6867..79778d287 100644 --- a/nativelink-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -24,6 +24,7 @@ use crate::serde_utils::{ convert_vec_string_with_shellexpand, }; use crate::stores::{ClientTlsConfig, ConfigDigestHashFunction, StoreRefName, StoreSpec}; +use crate::NamedConfig; /// Name of the scheduler. This type will be used when referencing a /// scheduler in the `CasConfig::schedulers`'s map key. @@ -725,7 +726,7 @@ pub struct GlobalConfig { pub struct CasConfig { /// List of stores available to use in this config. /// The keys can be used in other configs when needing to reference a store. - pub stores: HashMap, + pub stores: Vec>, /// Worker configurations used to execute jobs. pub workers: Option>, @@ -733,7 +734,7 @@ pub struct CasConfig { /// List of schedulers available to use in this config. /// The keys can be used in other configs when needing to reference a /// scheduler. - pub schedulers: Option>, + pub schedulers: Option>>, /// Servers to setup for this process. pub servers: Vec, diff --git a/nativelink-config/src/lib.rs b/nativelink-config/src/lib.rs index 0607e28c5..d2bee1e09 100644 --- a/nativelink-config/src/lib.rs +++ b/nativelink-config/src/lib.rs @@ -16,3 +16,12 @@ pub mod cas_server; pub mod schedulers; pub mod serde_utils; pub mod stores; + +use serde::Deserialize; + +#[derive(Debug, Clone, Deserialize)] +pub struct NamedConfig { + pub name: String, + #[serde(flatten)] + pub spec: Spec, +} diff --git a/src/bin/nativelink.rs b/src/bin/nativelink.rs index 42ae7096c..ae45232a5 100644 --- a/src/bin/nativelink.rs +++ b/src/bin/nativelink.rs @@ -29,7 +29,9 @@ use mimalloc::MiMalloc; use nativelink_config::cas_server::{ CasConfig, GlobalConfig, HttpCompressionAlgorithm, ListenerConfig, ServerConfig, WorkerConfig, }; -use nativelink_config::stores::ConfigDigestHashFunction; +use nativelink_config::schedulers::SchedulerSpec; +use nativelink_config::stores::{ConfigDigestHashFunction, StoreSpec}; +use nativelink_config::NamedConfig; use nativelink_error::{make_err, Code, Error, ResultExt}; use nativelink_metric::{ MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, RootMetricsComponent, @@ -183,11 +185,11 @@ async fn inner_main( { let mut health_registry_lock = health_registry_builder.lock().await; - for (name, store_cfg) in cfg.stores { + for NamedConfig:: { name, spec } in cfg.stores { let health_component_name = format!("stores/{name}"); let mut health_register_store = health_registry_lock.sub_builder(&health_component_name); - let store = store_factory(&store_cfg, &store_manager, Some(&mut health_register_store)) + let store = store_factory(&spec, &store_manager, Some(&mut health_register_store)) .await .err_tip(|| format!("Failed to create store '{name}'"))?; store_manager.add_store(&name, store); @@ -196,17 +198,15 @@ async fn inner_main( let mut action_schedulers = HashMap::new(); let mut worker_schedulers = HashMap::new(); - if let Some(schedulers_cfg) = cfg.schedulers { - for (name, scheduler_cfg) in schedulers_cfg { - let (maybe_action_scheduler, maybe_worker_scheduler) = - scheduler_factory(&scheduler_cfg, &store_manager) - .err_tip(|| format!("Failed to create scheduler '{name}'"))?; - if let Some(action_scheduler) = maybe_action_scheduler { - action_schedulers.insert(name.clone(), action_scheduler.clone()); - } - if let Some(worker_scheduler) = maybe_worker_scheduler { - worker_schedulers.insert(name.clone(), worker_scheduler.clone()); - } + for NamedConfig:: { name, spec } in cfg.schedulers.iter().flatten() { + let (maybe_action_scheduler, maybe_worker_scheduler) = + scheduler_factory(spec, &store_manager) + .err_tip(|| format!("Failed to create scheduler '{name}'"))?; + if let Some(action_scheduler) = maybe_action_scheduler { + action_schedulers.insert(name.clone(), action_scheduler.clone()); + } + if let Some(worker_scheduler) = maybe_worker_scheduler { + worker_schedulers.insert(name.clone(), worker_scheduler.clone()); } }