From da2c4a70662267b2f8e8992ea42a439a0e7ab2ec Mon Sep 17 00:00:00 2001 From: matdexir Date: Mon, 3 Jun 2024 14:13:41 +0800 Subject: [PATCH] Make quantity field human readable (#891) Add support for human readable quantity field in the different json config, i.e, 10KB -> 10000. This implementation still allows for the field being pure unsigned integers, so, it can be seen as offering an alternative configuration. Co-authored-by: Marcus Eagan --- Cargo.lock | 262 +++++++++++++++++- nativelink-config/BUILD.bazel | 19 ++ nativelink-config/Cargo.toml | 5 +- nativelink-config/src/cas_server.rs | 11 +- nativelink-config/src/lib.rs | 2 +- nativelink-config/src/schedulers.rs | 6 +- nativelink-config/src/serde_utils.rs | 80 ++++++ nativelink-config/src/stores.rs | 23 +- .../tests/deserialization_test.rs | 71 +++++ 9 files changed, 456 insertions(+), 23 deletions(-) create mode 100644 nativelink-config/tests/deserialization_test.rs diff --git a/Cargo.lock b/Cargo.lock index d2cee8a9d..928200043 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.11" @@ -257,7 +268,7 @@ version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2924dd7efd0112a5a88ec7d1c2dbf371966f018e90f3f45db7c8027ef895662a" dependencies = [ - "ahash", + "ahash 0.8.11", "aws-credential-types", "aws-runtime", "aws-sigv4", @@ -698,6 +709,18 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake3" version = "1.5.1" @@ -721,6 +744,63 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borsh" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbe5b10e214954177fb1dc9fbd20a1a2608fe99e6c832033bdc7cea287a20d77" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a8646f94ab393e43e8b35a2558b1624bed28b97ee09c5d15456e3c9463f46d" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.66", + "syn_derive", +] + +[[package]] +name = "byte-unit" +version = "5.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ac19bdf0b2665407c39d82dbc937e951e7e2001609f0fb32edd0af45a2d63e" +dependencies = [ + "rust_decimal", + "serde", + "utf8-width", +] + +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -755,6 +835,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "clap" version = "4.5.4" @@ -1178,6 +1264,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db0f0c49aba98a3b2578315766960bd242885ff672fd62610c5557cd6c6efe03" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.30" @@ -1329,6 +1421,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] [[package]] name = "hashbrown" @@ -1336,7 +1431,7 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash", + "ahash 0.8.11", "allocator-api2", ] @@ -1774,7 +1869,10 @@ dependencies = [ name = "nativelink-config" version = "0.4.0" dependencies = [ + "byte-unit", + "humantime", "serde", + "serde_json5", "shellexpand", ] @@ -2260,6 +2358,15 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2369,6 +2476,26 @@ dependencies = [ "prost", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "quote" version = "1.0.36" @@ -2378,6 +2505,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.5" @@ -2532,6 +2665,15 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "rfc6979" version = "0.3.1" @@ -2558,6 +2700,35 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta", + "rend", + "rkyv_derive", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.7.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "roxmltree" version = "0.14.1" @@ -2567,6 +2738,22 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "rust_decimal" +version = "1.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a" +dependencies = [ + "arrayvec", + "borsh", + "bytes", + "num-traits", + "rand", + "rkyv", + "serde", + "serde_json", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -2744,6 +2931,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b84345e4c9bd703274a082fb80caaa99b7612be48dfaa1dd9266577ec412309d" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "sec1" version = "0.3.0" @@ -2948,6 +3141,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "slab" version = "0.4.9" @@ -3029,12 +3228,30 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1329189c02ff984e9736652b1631330da25eaa6bc639089ed4915d25446cbe7b" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "sync_wrapper" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.10.1" @@ -3220,6 +3437,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" + +[[package]] +name = "toml_edit" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +dependencies = [ + "indexmap 2.2.6", + "toml_datetime", + "winnow", +] + [[package]] name = "tonic" version = "0.10.2" @@ -3470,6 +3704,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf8-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3" + [[package]] name = "utf8parse" version = "0.2.1" @@ -3686,6 +3926,24 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/nativelink-config/BUILD.bazel b/nativelink-config/BUILD.bazel index 51097f31e..a04ce4749 100644 --- a/nativelink-config/BUILD.bazel +++ b/nativelink-config/BUILD.bazel @@ -3,6 +3,7 @@ load( "rust_doc", "rust_doc_test", "rust_library", + "rust_test_suite", ) rust_library( @@ -16,11 +17,29 @@ rust_library( ], visibility = ["//visibility:public"], deps = [ + "@crates//:byte-unit", + "@crates//:humantime", "@crates//:serde", "@crates//:shellexpand", ], ) +rust_test_suite( + name = "integration", + timeout = "short", + srcs = [ + "tests/deserialization_test.rs", + ], + deps = [ + "//nativelink-config", + "//nativelink-error", + "@crates//:byte-unit", + "@crates//:humantime", + "@crates//:serde", + "@crates//:serde_json5", + ], +) + rust_doc( name = "docs", crate = ":nativelink-config", diff --git a/nativelink-config/Cargo.toml b/nativelink-config/Cargo.toml index d59d663db..f49de2098 100644 --- a/nativelink-config/Cargo.toml +++ b/nativelink-config/Cargo.toml @@ -4,5 +4,8 @@ version = "0.4.0" edition = "2021" [dependencies] -serde = { version = "1.0.201", features = ["derive"] } +byte-unit = "5.1.4" +humantime = "2.1.0" +serde = { version = "1.0.198", features = ["derive"] } +serde_json5 = "0.1.0" shellexpand = "3.1.0" diff --git a/nativelink-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs index a8461d6da..d46ff0004 100644 --- a/nativelink-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -18,6 +18,7 @@ use serde::Deserialize; use crate::schedulers::SchedulerConfig; use crate::serde_utils::{ + convert_data_size_with_shellexpand, convert_duration_with_shellexpand, convert_numeric_with_shellexpand, convert_optional_numeric_with_shellexpand, convert_optional_string_with_shellexpand, convert_string_with_shellexpand, convert_vec_string_with_shellexpand, @@ -139,7 +140,7 @@ pub struct ByteStreamConfig { /// 16KiB - 64KiB is optimal. /// /// Defaults: 64KiB - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub max_bytes_per_stream: usize, /// In the event a client disconnects while uploading a blob, we will hold @@ -148,7 +149,7 @@ pub struct ByteStreamConfig { /// the same blob. /// /// Defaults: 10 (seconds) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub persist_stream_on_disconnect_timeout: usize, } @@ -557,7 +558,7 @@ pub struct LocalWorkerConfig { /// longer than this time limit, the task will be rejected. Value in seconds. /// /// Default: 1200 (seconds / 20 mins) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub max_action_timeout: usize, /// If timeout is handled in `entrypoint` or another wrapper script. @@ -667,7 +668,7 @@ pub struct GlobalConfig { /// a new file descriptor because the limit has been reached. /// /// Default: 1000 (1 second) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub idle_file_descriptor_timeout_millis: u64, /// This flag can be used to prevent metrics from being collected at runtime. @@ -695,7 +696,7 @@ pub struct GlobalConfig { /// digest. /// /// Default: 1024*1024 (1MiB) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub default_digest_size_health_check: usize, } diff --git a/nativelink-config/src/lib.rs b/nativelink-config/src/lib.rs index b518c5ef1..866d9a9e6 100644 --- a/nativelink-config/src/lib.rs +++ b/nativelink-config/src/lib.rs @@ -14,5 +14,5 @@ pub mod cas_server; pub mod schedulers; -mod serde_utils; +pub mod serde_utils; pub mod stores; diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index ddd522baf..76f475734 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use serde::Deserialize; -use crate::serde_utils::convert_numeric_with_shellexpand; +use crate::serde_utils::{convert_duration_with_shellexpand, convert_numeric_with_shellexpand}; use crate::stores::{GrpcEndpoint, Retry, StoreRefName}; #[allow(non_camel_case_types)] @@ -97,13 +97,13 @@ pub struct SimpleScheduler { /// The amount of time to retain completed actions in memory for in case /// a WaitExecution is called after the action has completed. /// Default: 60 (seconds) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub retain_completed_for_s: u64, /// Remove workers from pool once the worker has not responded in this /// amount of time in seconds. /// Default: 5 (seconds) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub worker_timeout_s: u64, /// If a job returns an internal error or times out this many times when diff --git a/nativelink-config/src/serde_utils.rs b/nativelink-config/src/serde_utils.rs index 8fead477c..9d754e3dd 100644 --- a/nativelink-config/src/serde_utils.rs +++ b/nativelink-config/src/serde_utils.rs @@ -16,6 +16,8 @@ use std::fmt; use std::marker::PhantomData; use std::str::FromStr; +use byte_unit::Byte; +use humantime::parse_duration; use serde::{de, Deserialize, Deserializer}; /// Helper for serde macro so you can use shellexpand variables in the json configuration @@ -138,3 +140,81 @@ pub fn convert_optional_string_with_shellexpand<'de, D: Deserializer<'de>>( Ok(None) } } + +pub fn convert_data_size_with_shellexpand<'de, D, T, E>(deserializer: D) -> Result +where + D: Deserializer<'de>, + E: fmt::Display, + T: TryFrom + FromStr, + >::Error: fmt::Display, +{ + // define a visitor that deserializes + // `ActualData` encoded as json within a string + struct USizeVisitor>(PhantomData); + + impl<'de, T, FromStrErr> de::Visitor<'de> for USizeVisitor + where + FromStrErr: fmt::Display, + T: TryFrom + FromStr, + >::Error: fmt::Display, + { + type Value = T; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string containing json data") + } + + fn visit_i64(self, v: i64) -> Result { + v.try_into().map_err(de::Error::custom) + } + + fn visit_str(self, v: &str) -> Result { + let expanded = (*shellexpand::env(v).map_err(de::Error::custom)?).to_string(); + let byte_size = Byte::parse_str(expanded, true).map_err(de::Error::custom)?; + let byte_size_u128 = byte_size.as_u128(); + T::try_from(byte_size_u128.try_into().map_err(de::Error::custom)?) + .map_err(de::Error::custom) + } + } + + deserializer.deserialize_any(USizeVisitor::(PhantomData:: {})) +} + +pub fn convert_duration_with_shellexpand<'de, D, T, E>(deserializer: D) -> Result +where + D: Deserializer<'de>, + E: fmt::Display, + T: TryFrom + FromStr, + >::Error: fmt::Display, +{ + // define a visitor that deserializes + // `ActualData` encoded as json within a string + struct USizeVisitor>(PhantomData); + + impl<'de, T, FromStrErr> de::Visitor<'de> for USizeVisitor + where + FromStrErr: fmt::Display, + T: TryFrom + FromStr, + >::Error: fmt::Display, + { + type Value = T; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string containing json data") + } + + fn visit_i64(self, v: i64) -> Result { + v.try_into().map_err(de::Error::custom) + } + + fn visit_str(self, v: &str) -> Result { + let expanded = (*shellexpand::env(v).map_err(de::Error::custom)?).to_string(); + let duration = parse_duration(&expanded).map_err(de::Error::custom)?; + let duration_secs = duration.as_secs(); + T::try_from(duration_secs.try_into().map_err(de::Error::custom)?) + .map_err(de::Error::custom) + } + } + + deserializer.deserialize_any(USizeVisitor::(PhantomData:: {})) +} diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index e42432d9d..3614efc58 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -15,6 +15,7 @@ use serde::{Deserialize, Serialize}; use crate::serde_utils::{ + convert_data_size_with_shellexpand, convert_duration_with_shellexpand, convert_numeric_with_shellexpand, convert_optional_string_with_shellexpand, convert_string_with_shellexpand, convert_vec_string_with_shellexpand, }; @@ -203,7 +204,7 @@ pub struct ShardStore { #[serde(deny_unknown_fields)] pub struct SizePartitioningStore { /// Size to partition the data on. - #[serde(deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(deserialize_with = "convert_data_size_with_shellexpand")] pub size: u64, /// Store to send data when object is < (less than) size. @@ -243,7 +244,7 @@ pub struct FilesystemStore { /// Buffer size to use when reading files. Generally this should be left /// to the default value except for testing. /// Default: 32k. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub read_buffer_size: u32, /// Policy used to evict items out of the store. Failure to set this @@ -255,7 +256,7 @@ pub struct FilesystemStore { /// value is used to determine an entry's actual size on disk consumed /// For a 4KB block size filesystem, a 1B file actually consumes 4KB /// Default: 4096 - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub block_size: u64, } @@ -297,7 +298,7 @@ pub struct DedupStore { /// deciding where to partition the data. /// /// Default: 65536 (64k) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub min_size: u32, /// A best-effort attempt will be made to keep the average size @@ -311,13 +312,13 @@ pub struct DedupStore { /// details. /// /// Default: 262144 (256k) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub normal_size: u32, /// Maximum size a chunk is allowed to be. /// /// Default: 524288 (512k) - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub max_size: u32, /// Due to implementation detail, we want to prefer to download @@ -396,7 +397,7 @@ pub struct Lz4Config { /// compression ratios. /// /// Default: 65536 (64k). - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub block_size: u32, /// Maximum size allowed to attempt to deserialize data into. @@ -407,7 +408,7 @@ pub struct Lz4Config { /// allow you to specify the maximum that we'll attempt deserialize. /// /// Default: value in `block_size`. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub max_decode_block_size: u32, } @@ -447,19 +448,19 @@ pub struct CompressionStore { pub struct EvictionPolicy { /// Maximum number of bytes before eviction takes place. /// Default: 0. Zero means never evict based on size. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub max_bytes: usize, /// When eviction starts based on hitting max_bytes, continue until /// max_bytes - evict_bytes is met to create a low watermark. This stops /// operations from thrashing when the store is close to the limit. /// Default: 0 - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] pub evict_bytes: usize, /// Maximum number of seconds for an entry to live before an eviction. /// Default: 0. Zero means never evict based on time. - #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] pub max_seconds: u32, /// Maximum size of the store before an eviction takes place. diff --git a/nativelink-config/tests/deserialization_test.rs b/nativelink-config/tests/deserialization_test.rs new file mode 100644 index 000000000..2951676fc --- /dev/null +++ b/nativelink-config/tests/deserialization_test.rs @@ -0,0 +1,71 @@ +// Copyright 2023 The NativeLink Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use nativelink_config::serde_utils::{ + convert_data_size_with_shellexpand, convert_duration_with_shellexpand, +}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct DurationEntity { + #[serde(default, deserialize_with = "convert_duration_with_shellexpand")] + duration: usize, +} + +#[derive(Deserialize)] +struct DataSizeEntity { + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + data_size: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_duration_human_readable_deserialize() { + let example = r#" + {"duration": "1m 10s"} + "#; + let deserialized: DurationEntity = serde_json5::from_str(example).unwrap(); + assert_eq!(deserialized.duration, 70); + } + + #[test] + fn test_duration_usize_deserialize() { + let example = r#" + {"duration": 10} + "#; + let deserialized: DurationEntity = serde_json5::from_str(example).unwrap(); + assert_eq!(deserialized.duration, 10); + } + + #[test] + fn test_data_size_unit_deserialize() { + let example = r#" + {"data_size": "1KiB"} + "#; + let deserialized: DataSizeEntity = serde_json5::from_str(example).unwrap(); + assert_eq!(deserialized.data_size, 1024); + } + + #[test] + fn test_data_size_usize_deserialize() { + let example = r#" + {"data_size": 10} + "#; + let deserialized: DataSizeEntity = serde_json5::from_str(example).unwrap(); + assert_eq!(deserialized.data_size, 10); + } +}