From 3a84d72c266c0c0c99be2e1dec83e409f172515b Mon Sep 17 00:00:00 2001 From: Cass Fridkin Date: Fri, 23 Aug 2024 09:56:14 -0600 Subject: [PATCH] Replace usage of `deserialize_with` with `serde_as` --- nativelink-config/BUILD.bazel | 4 + nativelink-config/src/cas_server.rs | 151 +++++++++--------- nativelink-config/src/schedulers.rs | 13 +- nativelink-config/src/stores.rs | 88 ++++++---- .../tests/deserialization_test.rs | 13 +- 5 files changed, 157 insertions(+), 112 deletions(-) diff --git a/nativelink-config/BUILD.bazel b/nativelink-config/BUILD.bazel index d232a0cfc..56eb6ce3d 100644 --- a/nativelink-config/BUILD.bazel +++ b/nativelink-config/BUILD.bazel @@ -43,6 +43,7 @@ rust_test_suite( "@crates//:pretty_assertions", "@crates//:serde", "@crates//:serde_json5", + "@crates//:serde_with", ], ) @@ -65,4 +66,7 @@ rust_doc_test( name = "doc_test", timeout = "short", crate = ":nativelink-config", + deps = [ + "@crates//:serde_json5", + ], ) diff --git a/nativelink-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs index 40574379c..bc4405540 100644 --- a/nativelink-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -15,14 +15,10 @@ use std::collections::HashMap; use serde::Deserialize; +use serde_with::{serde_as, OneOrMany}; +use crate::deser::{ShellExpand, ShellExpandBytes, ShellExpandSeconds}; use crate::schedulers::SchedulerConfig; -use crate::serde_utils::{ - convert_data_size_with_shellexpand, convert_duration_with_shellexpand, - convert_numeric_with_shellexpand, convert_optional_numeric_with_shellexpand, - convert_optional_string_with_shellexpand, convert_string_with_shellexpand, - convert_vec_string_with_shellexpand, -}; use crate::stores::{ClientTlsConfig, ConfigDigestHashFunction, StoreConfig, StoreRefName}; /// Name of the scheduler. This type will be used when referencing a @@ -75,12 +71,13 @@ pub struct HttpCompressionConfig { pub accepted_compression_algorithms: Vec, } +#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct AcStoreConfig { /// The store name referenced in the `stores` map in the main config. /// This store name referenced here may be reused multiple times. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub ac_store: StoreRefName, /// Whether the Action Cache store may be written to, this if set to false @@ -89,20 +86,22 @@ pub struct AcStoreConfig { pub read_only: bool, } +#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct CasStoreConfig { /// The store name referenced in the `stores` map in the main config. /// This store name referenced here may be reused multiple times. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub cas_store: StoreRefName, } +#[serde_as] #[derive(Deserialize, Debug, Default)] #[serde(deny_unknown_fields)] pub struct CapabilitiesRemoteExecutionConfig { /// Scheduler used to configure the capabilities of remote execution. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub scheduler: SchedulerRefName, } @@ -115,20 +114,22 @@ pub struct CapabilitiesConfig { pub remote_execution: Option, } +#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct ExecutionConfig { /// The store name referenced in the `stores` map in the main config. /// This store name referenced here may be reused multiple times. /// This value must be a CAS store reference. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub cas_store: StoreRefName, /// The scheduler name referenced in the `schedulers` map in the main config. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub scheduler: SchedulerRefName, } +#[serde_as] #[derive(Deserialize, Debug, Default)] #[serde(deny_unknown_fields)] pub struct ByteStreamConfig { @@ -141,12 +142,14 @@ pub struct ByteStreamConfig { /// /// /// Default: 64KiB - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub max_bytes_per_stream: usize, /// Maximum number of bytes to decode on each grpc stream chunk. /// Default: 4 MiB - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub max_decoding_message_size: usize, /// In the event a client disconnects while uploading a blob, we will hold @@ -155,15 +158,17 @@ pub struct ByteStreamConfig { /// the same blob. /// /// Default: 10 (seconds) - #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + #[serde_as(as = "ShellExpandSeconds")] + #[serde(default)] pub persist_stream_on_disconnect_timeout: usize, } +#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct WorkerApiConfig { /// The scheduler name referenced in the `schedulers` map in the main config. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub scheduler: SchedulerRefName, } @@ -203,11 +208,12 @@ pub struct HealthConfig { pub path: String, } +#[serde_as] #[derive(Deserialize, Debug)] pub struct BepConfig { /// The store to publish build events to. /// The store name referenced in the `stores` map in the main config. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub store: StoreRefName, } @@ -265,25 +271,28 @@ pub struct ServicesConfig { pub health: Option, } +#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct TlsConfig { /// Path to the certificate file. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub cert_file: String, /// Path to the private key file. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub key_file: String, /// Path to the certificate authority for mTLS, if client authentication is /// required for this endpoint. - #[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")] + #[serde_as(as = "Option")] + #[serde(default)] pub client_ca_file: Option, /// Path to the certificate revocation list for mTLS, if client /// authentication is required for this endpoint. - #[serde(default, deserialize_with = "convert_optional_string_with_shellexpand")] + #[serde_as(as = "Option")] + #[serde(default)] pub client_crl_file: Option, } @@ -293,70 +302,53 @@ pub struct TlsConfig { /// /// Note: All of these default to hyper's default values unless otherwise /// specified. +#[serde_as] #[derive(Deserialize, Debug, Default)] #[serde(deny_unknown_fields)] pub struct HttpServerConfig { /// Interval to send keep-alive pings via HTTP2. /// Note: This is in seconds. - #[serde( - default, - deserialize_with = "convert_optional_numeric_with_shellexpand" - )] + #[serde_as(as = "Option")] + #[serde(default)] pub http2_keep_alive_interval: Option, - #[serde( - default, - deserialize_with = "convert_optional_numeric_with_shellexpand" - )] + #[serde_as(as = "Option")] + #[serde(default)] pub experimental_http2_max_pending_accept_reset_streams: Option, - #[serde( - default, - deserialize_with = "convert_optional_numeric_with_shellexpand" - )] + #[serde_as(as = "Option")] + #[serde(default)] pub experimental_http2_initial_stream_window_size: Option, - #[serde( - default, - deserialize_with = "convert_optional_numeric_with_shellexpand" - )] + #[serde_as(as = "Option")] + #[serde(default)] pub experimental_http2_initial_connection_window_size: Option, #[serde(default)] pub experimental_http2_adaptive_window: Option, - #[serde( - default, - deserialize_with = "convert_optional_numeric_with_shellexpand" - )] + #[serde_as(as = "Option")] + #[serde(default)] pub experimental_http2_max_frame_size: Option, - #[serde( - default, - deserialize_with = "convert_optional_numeric_with_shellexpand" - )] + #[serde_as(as = "Option")] + #[serde(default)] pub experimental_http2_max_concurrent_streams: Option, /// Note: This is in seconds. - #[serde( - default, - deserialize_with = "convert_optional_numeric_with_shellexpand" - )] + #[serde_as(as = "Option")] + #[serde(default)] pub experimental_http2_keep_alive_timeout: Option, - #[serde( - default, - deserialize_with = "convert_optional_numeric_with_shellexpand" - )] + #[serde_as(as = "Option")] + #[serde(default)] pub experimental_http2_max_send_buf_size: Option, #[serde(default)] pub experimental_http2_enable_connect_protocol: Option, - #[serde( - default, - deserialize_with = "convert_optional_numeric_with_shellexpand" - )] + #[serde_as(as = "Option")] + #[serde(default)] pub experimental_http2_max_header_list_size: Option, } @@ -367,12 +359,13 @@ pub enum ListenerConfig { Http(HttpListener), } +#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct HttpListener { /// Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen /// to all IPs. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub socket_address: String, /// Data transport compression configuration to use for this service. @@ -391,6 +384,7 @@ pub struct HttpListener { pub tls: Option, } +#[serde_as] #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct ServerConfig { @@ -398,7 +392,8 @@ pub struct ServerConfig { /// for telemetry and logs. /// /// Default: {index of server in config} - #[serde(default, deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub name: String, /// Configuration @@ -408,14 +403,14 @@ pub struct ServerConfig { pub services: Option, } +#[serde_as] #[derive(Deserialize, Debug)] #[serde(rename_all = "snake_case")] pub enum WorkerProperty { /// List of static values. /// Note: Generally there should only ever be 1 value, but if the platform /// property key is PropertyType::Priority it may have more than one value. - #[serde(deserialize_with = "convert_vec_string_with_shellexpand")] - Values(Vec), + Values(#[serde_as(as = "OneOrMany")] Vec), /// A dynamic configuration. The string will be executed as a command /// (not sell) and will be split by "\n" (new line character). @@ -423,11 +418,12 @@ pub enum WorkerProperty { } /// Generic config for an endpoint and associated configs. +#[serde_as] #[derive(Deserialize, Debug, Default)] #[serde(deny_unknown_fields)] pub struct EndpointConfig { /// URI of the endpoint. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub uri: String, /// Timeout in seconds that a request should take. @@ -455,6 +451,7 @@ pub enum UploadCacheResultsStrategy { FailuresOnly, } +#[serde_as] #[derive(Clone, Deserialize, Debug)] #[serde(rename_all = "snake_case")] pub enum EnvironmentSource { @@ -462,7 +459,7 @@ pub enum EnvironmentSource { Property(String), /// The raw value to set. - Value(#[serde(deserialize_with = "convert_string_with_shellexpand")] String), + Value(#[serde_as(as = "ShellExpand")] String), /// The max amount of time in milliseconds the command is allowed to run /// (requested by the client). @@ -502,6 +499,7 @@ pub enum EnvironmentSource { ActionDirectory, } +#[serde_as] #[derive(Deserialize, Debug, Default)] #[serde(deny_unknown_fields)] pub struct UploadActionResultConfig { @@ -549,7 +547,8 @@ pub struct UploadActionResultConfig { /// /// /// Default: "" (no message) - #[serde(default, deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub success_message_template: String, /// Same as `success_message_template` but for failure case. @@ -558,17 +557,20 @@ pub struct UploadActionResultConfig { /// /// /// Default: "" (no message) - #[serde(default, deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub failure_message_template: String, } +#[serde_as] #[derive(Deserialize, Debug, Default)] #[serde(deny_unknown_fields)] pub struct LocalWorkerConfig { /// Name of the worker. This is give a more friendly name to a worker for logging /// and metric publishing. /// Default: {Index position in the workers list} - #[serde(default, deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub name: String, /// Endpoint which the worker will connect to the scheduler's WorkerApiService. @@ -578,7 +580,8 @@ pub struct LocalWorkerConfig { /// longer than this time limit, the task will be rejected. Value in seconds. /// /// Default: 1200 (seconds / 20 mins) - #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + #[serde_as(as = "ShellExpandSeconds")] + #[serde(default)] pub max_action_timeout: usize, /// If timeout is handled in `entrypoint` or another wrapper script. @@ -606,7 +609,8 @@ pub struct LocalWorkerConfig { /// Example: "run.sh" and a job with command: "sleep 5" will result in a /// command like: "run.sh sleep 5". /// Default: {Use the command from the job request}. - #[serde(default, deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub entrypoint: String, /// An optional script to run before every action is processed on the worker. @@ -624,7 +628,7 @@ pub struct LocalWorkerConfig { /// `FileSystemStore` because it will use hardlinks when building out the files /// instead of copying the files. The slow store must eventually resolve to the /// same store the scheduler/client uses to send job requests. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub cas_fast_slow_store: StoreRefName, /// Configuration for uploading action results. @@ -637,7 +641,7 @@ pub struct LocalWorkerConfig { /// stores::FilesystemStore::content_path must be on the same filesystem. /// Hardlinks will be used when placing files that are accessible to the jobs /// that are sourced from local_filesystem_store_ref's content_path. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub work_directory: String, /// Properties of this worker. This configuration will be sent to the scheduler @@ -660,6 +664,7 @@ pub enum WorkerConfig { Local(LocalWorkerConfig), } +#[serde_as] #[derive(Deserialize, Debug, Clone, Copy)] #[serde(deny_unknown_fields)] pub struct GlobalConfig { @@ -673,7 +678,7 @@ pub struct GlobalConfig { /// Note: This value must be greater than 10. /// /// Default: 512 - #[serde(deserialize_with = "convert_numeric_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub max_open_files: usize, /// If a file descriptor is idle for this many milliseconds, it will be closed. @@ -688,7 +693,8 @@ pub struct GlobalConfig { /// a new file descriptor because the limit has been reached. /// /// Default: 1000 (1 second) - #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + #[serde_as(as = "ShellExpandSeconds")] + #[serde(default)] pub idle_file_descriptor_timeout_millis: u64, /// This flag can be used to prevent metrics from being collected at runtime. @@ -716,7 +722,8 @@ pub struct GlobalConfig { /// digest. /// /// Default: 1024*1024 (1MiB) - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub default_digest_size_health_check: usize, } diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index d33002190..87b5aacd4 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -15,8 +15,9 @@ use std::collections::HashMap; use serde::Deserialize; +use serde_with::serde_as; -use crate::serde_utils::{convert_duration_with_shellexpand, convert_numeric_with_shellexpand}; +use crate::deser::{ShellExpand, ShellExpandSeconds}; use crate::stores::{GrpcEndpoint, Retry, StoreRefName}; #[derive(Deserialize, Debug)] @@ -65,6 +66,7 @@ pub enum WorkerAllocationStrategy { MostRecentlyUsed, } +#[serde_as] #[derive(Deserialize, Debug, Default)] #[serde(deny_unknown_fields)] pub struct SimpleScheduler { @@ -97,13 +99,15 @@ pub struct SimpleScheduler { /// The amount of time to retain completed actions in memory for in case /// a WaitExecution is called after the action has completed. /// Default: 60 (seconds) - #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + #[serde_as(as = "ShellExpandSeconds")] + #[serde(default)] pub retain_completed_for_s: u32, /// Remove workers from pool once the worker has not responded in this /// amount of time in seconds. /// Default: 5 (seconds) - #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + #[serde_as(as = "ShellExpandSeconds")] + #[serde(default)] pub worker_timeout_s: u64, /// If a job returns an internal error or times out this many times when @@ -113,7 +117,8 @@ pub struct SimpleScheduler { /// resources when the task itself is the one causing the server to go /// into a bad state. /// Default: 3 - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub max_job_retries: usize, /// The strategy used to assign workers jobs. diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index d4a9e6af6..a22d0e708 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -13,12 +13,9 @@ // limitations under the License. use serde::{Deserialize, Serialize}; +use serde_with::serde_as; -use crate::serde_utils::{ - convert_data_size_with_shellexpand, convert_duration_with_shellexpand, - convert_numeric_with_shellexpand, convert_optional_string_with_shellexpand, - convert_string_with_shellexpand, convert_vec_string_with_shellexpand, -}; +use crate::deser::{ShellExpand, ShellExpandBytes, ShellExpandSeconds}; /// Name of the store. This type will be used when referencing a store /// in the `CasConfig::stores`'s map key. @@ -456,11 +453,12 @@ pub struct ShardStore { pub stores: Vec, } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] pub struct SizePartitioningStore { /// Size to partition the data on. - #[serde(deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] pub size: u64, /// Store to send data when object is < (less than) size. @@ -470,14 +468,16 @@ pub struct SizePartitioningStore { pub upper_store: StoreConfig, } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Default, Clone)] #[serde(deny_unknown_fields)] pub struct RefStore { /// Name of the store under the root "stores" config object. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub name: String, } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Default, Clone)] #[serde(deny_unknown_fields)] pub struct FilesystemStore { @@ -486,7 +486,7 @@ pub struct FilesystemStore { /// On service bootup this folder will be scanned and all files will be /// added to the cache. In the event one of the files doesn't match the /// criteria, the file will be deleted. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub content_path: String, /// A temporary location of where files that are being uploaded or @@ -494,13 +494,14 @@ pub struct FilesystemStore { /// accurate. This location must be on the same block device as /// `content_path` so atomic moves can happen (ie: move without copy). /// All files in this folder will be deleted on every startup. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub temp_path: String, /// Buffer size to use when reading files. Generally this should be left /// to the default value except for testing. /// Default: 32k. - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub read_buffer_size: u32, /// Policy used to evict items out of the store. Failure to set this @@ -512,7 +513,8 @@ pub struct FilesystemStore { /// value is used to determine an entry's actual size on disk consumed /// For a 4KB block size filesystem, a 1B file actually consumes 4KB /// Default: 4096 - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub block_size: u64, } @@ -537,6 +539,7 @@ pub struct MemoryStore { pub eviction_policy: Option, } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] pub struct DedupStore { @@ -554,7 +557,8 @@ pub struct DedupStore { /// deciding where to partition the data. /// /// Default: 65536 (64k) - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub min_size: u32, /// A best-effort attempt will be made to keep the average size @@ -568,13 +572,15 @@ pub struct DedupStore { /// details. /// /// Default: 262144 (256k) - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub normal_size: u32, /// Maximum size a chunk is allowed to be. /// /// Default: 524288 (512k) - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub max_size: u32, /// Due to implementation detail, we want to prefer to download @@ -588,7 +594,8 @@ pub struct DedupStore { /// request is: `max_concurrent_fetch_per_get * max_size`. /// /// Default: 10 - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub max_concurrent_fetch_per_get: u32, } @@ -645,6 +652,7 @@ pub struct CompletenessCheckingStore { pub cas_store: StoreConfig, } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone, Copy)] #[serde(deny_unknown_fields)] pub struct Lz4Config { @@ -653,7 +661,8 @@ pub struct Lz4Config { /// compression ratios. /// /// Default: 65536 (64k). - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub block_size: u32, /// Maximum size allowed to attempt to deserialize data into. @@ -664,7 +673,8 @@ pub struct Lz4Config { /// allow you to specify the maximum that we'll attempt deserialize. /// /// Default: value in `block_size`. - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub max_decode_block_size: u32, } @@ -699,42 +709,50 @@ pub struct CompressionStore { /// is touched it updates the timestamp. Inserts and updates will execute the /// eviction policy removing any expired entries and/or the oldest entries /// until the store size becomes smaller than max_bytes. +#[serde_as] #[derive(Serialize, Deserialize, Debug, Default, Clone)] #[serde(deny_unknown_fields)] pub struct EvictionPolicy { /// Maximum number of bytes before eviction takes place. /// Default: 0. Zero means never evict based on size. - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub max_bytes: usize, /// When eviction starts based on hitting max_bytes, continue until /// max_bytes - evict_bytes is met to create a low watermark. This stops /// operations from thrashing when the store is close to the limit. /// Default: 0 - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] pub evict_bytes: usize, /// Maximum number of seconds for an entry to live since it was last /// accessed before it is evicted. /// Default: 0. Zero means never evict based on time. - #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + #[serde_as(as = "ShellExpandSeconds")] + #[serde(default)] pub max_seconds: u32, /// Maximum size of the store before an eviction takes place. /// Default: 0. Zero means never evict based on count. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub max_count: u64, } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Default, Clone)] #[serde(deny_unknown_fields)] pub struct S3Store { /// S3 region. Usually us-east-1, us-west-2, af-south-1, exc... - #[serde(default, deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub region: String, /// Bucket name to use as the backend. - #[serde(default, deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub bucket: String, /// If you wish to prefix the location on s3. If None, no prefix will be used. @@ -756,7 +774,8 @@ pub struct S3Store { /// around for a few days is generally a good idea. /// /// Default: 0. Zero means never consider an object expired. - #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + #[serde_as(as = "ShellExpandSeconds")] + #[serde(default)] pub consider_expired_after_s: u32, /// The maximum buffer size to retain in case of a retryable error @@ -798,26 +817,28 @@ pub enum StoreType { Ac, } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Clone)] pub struct ClientTlsConfig { /// Path to the certificate authority to use to validate the remote. - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub ca_file: String, /// Path to the certificate file for client authentication. - #[serde(deserialize_with = "convert_optional_string_with_shellexpand")] + #[serde_as(as = "Option")] pub cert_file: Option, /// Path to the private key file for client authentication. - #[serde(deserialize_with = "convert_optional_string_with_shellexpand")] + #[serde_as(as = "Option")] pub key_file: Option, } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] pub struct GrpcEndpoint { /// The endpoint address (i.e. grpc(s)://example.com:443). - #[serde(deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] pub address: String, /// The TLS configuration to use to connect to the endpoint (if grpcs). pub tls_config: Option, @@ -825,11 +846,13 @@ pub struct GrpcEndpoint { pub concurrency_limit: Option, } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] pub struct GrpcStore { /// Instance name for GRPC calls. Proxy calls will have the instance_name changed to this. - #[serde(default, deserialize_with = "convert_string_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub instance_name: String, /// The endpoint of the grpc connection. @@ -876,12 +899,13 @@ pub enum ErrorCode { // Note: This list is duplicated from nativelink-error/lib.rs. } +#[serde_as] #[derive(Serialize, Deserialize, Debug, Clone)] pub struct RedisStore { /// The hostname or IP address of the Redis server. /// Ex: ["redis://username:password@redis-server-url:6380/99"] /// 99 Represents database ID, 6380 represents the port. - #[serde(deserialize_with = "convert_vec_string_with_shellexpand")] + #[serde_as(as = "Vec")] pub addresses: Vec, /// The response timeout for the Redis connection in seconds. @@ -958,12 +982,14 @@ pub enum RedisMode { /// 8 4.8s - 8s /// Remember that to get total results is additive, meaning the above results /// would mean a single request would have a total delay of 9.525s - 15.875s. +#[serde_as] #[derive(Serialize, Deserialize, Clone, Debug, Default)] #[serde(deny_unknown_fields)] pub struct Retry { /// Maximum number of retries until retrying stops. /// Setting this to zero will always attempt 1 time, but not retry. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde_as(as = "ShellExpand")] + #[serde(default)] pub max_retries: usize, /// Delay in seconds for exponential back off. diff --git a/nativelink-config/tests/deserialization_test.rs b/nativelink-config/tests/deserialization_test.rs index 872600893..060b028d3 100644 --- a/nativelink-config/tests/deserialization_test.rs +++ b/nativelink-config/tests/deserialization_test.rs @@ -12,21 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -use nativelink_config::serde_utils::{ - convert_data_size_with_shellexpand, convert_duration_with_shellexpand, -}; +use nativelink_config::deser::{ShellExpandBytes, ShellExpandSeconds}; use pretty_assertions::assert_eq; use serde::Deserialize; +use serde_with::serde_as; +#[serde_as] #[derive(Deserialize)] struct DurationEntity { - #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + #[serde_as(as = "ShellExpandSeconds")] + #[serde(default)] duration: usize, } +#[serde_as] #[derive(Deserialize)] struct DataSizeEntity { - #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + #[serde_as(as = "ShellExpandBytes")] + #[serde(default)] data_size: usize, }