diff --git a/Cargo.lock b/Cargo.lock index 093061216a..f9ae4c29c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,7 +166,7 @@ dependencies = [ "omicron-workspace-hack", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -273,7 +273,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -295,7 +295,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -306,7 +306,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -359,7 +359,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -518,7 +518,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.64", + "syn 2.0.68", "which", ] @@ -1038,7 +1038,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1530,7 +1530,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1554,7 +1554,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1565,7 +1565,7 @@ checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" dependencies = [ "darling_core", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1599,7 +1599,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1642,7 +1642,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1675,7 +1675,7 @@ checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1696,7 +1696,7 @@ checksum = "62d671cc41a825ebabc75757b62d3d168c577f9149b2d49ece1dad1f72119d25" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1717,7 +1717,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1727,7 +1727,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1804,7 +1804,7 @@ dependencies = [ "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -1813,7 +1813,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -2071,7 +2071,7 @@ dependencies = [ "quote", "serde", "serde_tokenstream", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -2486,7 +2486,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -2598,7 +2598,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -3887,14 +3887,14 @@ version = "0.1.0" source = "git+https://github.com/oxidecomputer/opte?rev=417f74e94978c23f3892ac328c3387f3ecd9bb29#417f74e94978c23f3892ac328c3387f3ecd9bb29" dependencies = [ "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] name = "kstat-rs" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc713c7902f757cf0c04012dbad3864ea505f2660467b704847ea7ea2ff6d67" +checksum = "27964e4632377753acb0898ce6f28770d50cbca1339200ae63d700cff97b5c2b" dependencies = [ "libc", "thiserror", @@ -4376,7 +4376,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -4739,7 +4739,7 @@ dependencies = [ "omicron-workspace-hack", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -4935,7 +4935,7 @@ version = "0.1.0" dependencies = [ "omicron-workspace-hack", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -5106,7 +5106,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -5962,7 +5962,7 @@ dependencies = [ "string_cache", "subtle", "syn 1.0.109", - "syn 2.0.64", + "syn 2.0.68", "time", "time-macros", "tokio", @@ -6082,7 +6082,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -6225,24 +6225,16 @@ dependencies = [ name = "oximeter" version = "0.1.0" dependencies = [ - "approx", - "bytes", + "anyhow", "chrono", - "float-ord", - "num", - "omicron-common", + "clap", "omicron-workspace-hack", + "oximeter-impl", "oximeter-macro-impl", - "rand 0.8.5", - "rand_distr", - "regex", - "rstest", - "schemars", - "serde", - "serde_json", - "strum", - "thiserror", - "trybuild", + "oximeter-timeseries-macro", + "prettyplease", + "syn 2.0.68", + "toml 0.8.13", "uuid", ] @@ -6348,6 +6340,38 @@ dependencies = [ "uuid", ] +[[package]] +name = "oximeter-impl" +version = "0.1.0" +dependencies = [ + "approx", + "bytes", + "chrono", + "float-ord", + "heck 0.5.0", + "num", + "omicron-common", + "omicron-workspace-hack", + "oximeter-macro-impl", + "prettyplease", + "proc-macro2", + "quote", + "rand 0.8.5", + "rand_distr", + "regex", + "rstest", + "schemars", + "serde", + "serde_json", + "slog-error-chain", + "strum", + "syn 2.0.68", + "thiserror", + "toml 0.8.13", + "trybuild", + "uuid", +] + [[package]] name = "oximeter-instruments" version = "0.1.0" @@ -6358,6 +6382,7 @@ dependencies = [ "futures", "http 0.2.12", "kstat-rs", + "libc", "omicron-workspace-hack", "oximeter", "rand 0.8.5", @@ -6376,7 +6401,7 @@ dependencies = [ "omicron-workspace-hack", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -6404,6 +6429,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "oximeter-timeseries-macro" +version = "0.1.0" +dependencies = [ + "omicron-workspace-hack", + "oximeter-impl", + "proc-macro2", + "quote", + "syn 2.0.68", +] + [[package]] name = "oxlog" version = "0.1.0" @@ -6557,7 +6593,7 @@ dependencies = [ "regex", "regex-syntax 0.8.3", "structmeta 0.3.0", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -6725,7 +6761,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -6795,7 +6831,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -7065,7 +7101,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -7113,9 +7149,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.82" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -7161,7 +7197,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn 2.0.64", + "syn 2.0.68", "thiserror", "typify", "unicode-ident", @@ -7181,7 +7217,7 @@ dependencies = [ "serde_json", "serde_tokenstream", "serde_yaml", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -7671,7 +7707,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -7925,7 +7961,7 @@ dependencies = [ "regex", "relative-path", "rustc_version 0.4.0", - "syn 2.0.64", + "syn 2.0.68", "unicode-ident", ] @@ -8306,9 +8342,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0218ceea14babe24a4a5836f86ade86c1effbc198164e619194cb5069187e29" +checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" dependencies = [ "bytes", "chrono", @@ -8321,14 +8357,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed5a1ccce8ff962e31a165d41f6e2a2dd1245099dc4d594f5574a86cd90f4d3" +checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -8354,7 +8390,7 @@ checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -8483,7 +8519,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -8494,7 +8530,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -8544,7 +8580,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -8565,7 +8601,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -8607,7 +8643,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -8955,7 +8991,7 @@ source = "git+https://github.com/oxidecomputer/slog-error-chain?branch=main#15f6 dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9082,7 +9118,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9209,7 +9245,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9219,7 +9255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ff9eaf853dec4c8802325d8b6d3dffa86cc707fd7a1a4cdbf416e13b061787a" dependencies = [ "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9305,7 +9341,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive 0.2.0", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9317,7 +9353,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive 0.3.0", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9328,7 +9364,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9339,7 +9375,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9374,7 +9410,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9387,7 +9423,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9434,9 +9470,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.64" +version = "2.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ad3dee41f36859875573074334c200d1add8e4a87bb37113ebd31d926b7b11f" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" dependencies = [ "proc-macro2", "quote", @@ -9610,7 +9646,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta 0.2.0", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9641,7 +9677,7 @@ checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9778,7 +9814,7 @@ checksum = "8d9ef545650e79f30233c0003bcc2504d7efac6dad25fca40744de773fe2049c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -9849,7 +9885,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -10126,7 +10162,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -10403,7 +10439,7 @@ dependencies = [ "semver 1.0.23", "serde", "serde_json", - "syn 2.0.64", + "syn 2.0.68", "thiserror", "unicode-ident", ] @@ -10420,7 +10456,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.64", + "syn 2.0.68", "typify-impl", ] @@ -10627,7 +10663,7 @@ dependencies = [ "proc-macro2", "quote", "serde_tokenstream", - "syn 2.0.64", + "syn 2.0.68", "usdt-impl", ] @@ -10645,7 +10681,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.64", + "syn 2.0.68", "thiserror", "thread-id", "version_check", @@ -10661,7 +10697,7 @@ dependencies = [ "proc-macro2", "quote", "serde_tokenstream", - "syn 2.0.64", + "syn 2.0.68", "usdt-impl", ] @@ -10840,7 +10876,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", "wasm-bindgen-shared", ] @@ -10874,7 +10910,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -11456,7 +11492,7 @@ checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -11467,7 +11503,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] @@ -11487,7 +11523,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.64", + "syn 2.0.68", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1291f18726..3ba353c220 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,10 +59,12 @@ members = [ "nexus/types", "oximeter/collector", "oximeter/db", + "oximeter/impl", "oximeter/instruments", "oximeter/oximeter-macro-impl", "oximeter/oximeter", "oximeter/producer", + "oximeter/timeseries-macro", "package", "passwords", "rpaths", @@ -149,10 +151,12 @@ default-members = [ "nexus/types", "oximeter/collector", "oximeter/db", + "oximeter/impl", "oximeter/instruments", "oximeter/oximeter-macro-impl", "oximeter/oximeter", "oximeter/producer", + "oximeter/timeseries-macro", "package", "passwords", "rpaths", @@ -320,7 +324,7 @@ internet-checksum = "0.2" ipnetwork = { version = "0.20", features = ["schemars"] } ispf = { git = "https://github.com/oxidecomputer/ispf" } key-manager = { path = "key-manager" } -kstat-rs = "0.2.3" +kstat-rs = "0.2.4" libc = "0.2.155" libfalcon = { git = "https://github.com/oxidecomputer/falcon", rev = "e69694a1f7cc9fe31fab27f321017280531fb5f7" } libnvme = { git = "https://github.com/oxidecomputer/libnvme", rev = "6fffcc81d2c423ed2d2e6c5c2827485554c4ecbe" } @@ -382,9 +386,11 @@ oximeter = { path = "oximeter/oximeter" } oximeter-client = { path = "clients/oximeter-client" } oximeter-db = { path = "oximeter/db/" } oximeter-collector = { path = "oximeter/collector" } +oximeter-impl = { path = "oximeter/impl" } oximeter-instruments = { path = "oximeter/instruments" } oximeter-macro-impl = { path = "oximeter/oximeter-macro-impl" } oximeter-producer = { path = "oximeter/producer" } +oximeter-timeseries-macro = { path = "oximeter/timeseries-macro" } p256 = "0.13" parse-display = "0.9.0" partial-io = { version = "0.5.4", features = ["proptest1", "tokio1"] } diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 81cf6499b2..4d55a134c1 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -135,7 +135,7 @@ rcgen.workspace = true regex.workspace = true similar-asserts.workspace = true sp-sim.workspace = true -rustls = { workspace = true } +rustls.workspace = true subprocess.workspace = true term.workspace = true trust-dns-resolver.workspace = true diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 5d175e7b09..6050939b94 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -328,6 +328,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -340,6 +341,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -437,6 +439,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -444,11 +447,13 @@ "md5_auth_key": { "nullable": true, "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": "string" }, "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -456,6 +461,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -467,6 +473,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -474,6 +481,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -1192,6 +1200,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -1234,6 +1243,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index c3cc3c059d..72731e83e8 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -1567,6 +1567,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -1579,6 +1580,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -1676,6 +1678,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1683,11 +1686,13 @@ "md5_auth_key": { "nullable": true, "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": "string" }, "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -1695,6 +1700,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1706,6 +1712,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1713,6 +1720,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -4345,6 +4353,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -5003,6 +5012,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/openapi/nexus.json b/openapi/nexus.json index ab7192d1e0..a985a3e42c 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -9344,6 +9344,39 @@ } ] }, + "AuthzScope": { + "description": "Authorization scope for a timeseries.\n\nThis describes the level at which a user must be authorized to read data from a timeseries. For example, fleet-scoping means the data is only visible to an operator or fleet reader. Project-scoped, on the other hand, indicates that a user will see data limited to the projects on which they have read permissions.", + "oneOf": [ + { + "description": "Timeseries data is limited to fleet readers.", + "type": "string", + "enum": [ + "fleet" + ] + }, + { + "description": "Timeseries data is limited to the authorized silo for a user.", + "type": "string", + "enum": [ + "silo" + ] + }, + { + "description": "Timeseries data is limited to the authorized projects for a user.", + "type": "string", + "enum": [ + "project" + ] + }, + { + "description": "The timeseries is viewable to all without limitation.", + "type": "string", + "enum": [ + "viewable_to_all" + ] + } + ] + }, "Baseboard": { "description": "Properties that uniquely identify an Oxide hardware component", "type": "object", @@ -12646,6 +12679,9 @@ "description": "The name and type information for a field of a timeseries schema.", "type": "object", "properties": { + "description": { + "type": "string" + }, "field_type": { "$ref": "#/components/schemas/FieldType" }, @@ -12657,6 +12693,7 @@ } }, "required": [ + "description", "field_type", "name", "source" @@ -16686,6 +16723,7 @@ "signing_keypair": { "nullable": true, "description": "request signing key pair", + "default": null, "allOf": [ { "$ref": "#/components/schemas/DerEncodedKeyPair" @@ -18546,6 +18584,22 @@ "points" ] }, + "TimeseriesDescription": { + "description": "Text descriptions for the target and metric of a timeseries.", + "type": "object", + "properties": { + "metric": { + "type": "string" + }, + "target": { + "type": "string" + } + }, + "required": [ + "metric", + "target" + ] + }, "TimeseriesName": { "title": "The name of a timeseries", "description": "Names are constructed by concatenating the target and metric names with ':'. Target and metric names must be lowercase alphanumeric characters with '_' separating words.", @@ -18569,6 +18623,9 @@ "description": "The schema for a timeseries.\n\nThis includes the name of the timeseries, as well as the datum type of its metric and the schema for each field.", "type": "object", "properties": { + "authz_scope": { + "$ref": "#/components/schemas/AuthzScope" + }, "created": { "type": "string", "format": "date-time" @@ -18576,6 +18633,9 @@ "datum_type": { "$ref": "#/components/schemas/DatumType" }, + "description": { + "$ref": "#/components/schemas/TimeseriesDescription" + }, "field_schema": { "type": "array", "items": { @@ -18585,13 +18645,25 @@ }, "timeseries_name": { "$ref": "#/components/schemas/TimeseriesName" + }, + "units": { + "$ref": "#/components/schemas/Units" + }, + "version": { + "type": "integer", + "format": "uint8", + "minimum": 1 } }, "required": [ + "authz_scope", "created", "datum_type", + "description", "field_schema", - "timeseries_name" + "timeseries_name", + "units", + "version" ] }, "TimeseriesSchemaResultsPage": { @@ -18675,6 +18747,14 @@ "items" ] }, + "Units": { + "description": "Measurement units for timeseries samples.", + "type": "string", + "enum": [ + "count", + "bytes" + ] + }, "User": { "description": "View of a User", "type": "object", diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 21967f9fc3..05081d8298 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -1462,6 +1462,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -1474,6 +1475,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -1571,6 +1573,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1578,11 +1581,13 @@ "md5_auth_key": { "nullable": true, "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": "string" }, "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -1590,6 +1595,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1601,6 +1607,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1608,6 +1615,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -4247,6 +4255,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -4554,6 +4563,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 21e8ebeedd..555b8cf44c 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -1049,6 +1049,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -1061,6 +1062,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -2854,6 +2856,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -5050,6 +5053,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -5070,6 +5074,7 @@ }, "allowed_export": { "description": "Apply export policy to this peer with an allow list.", + "default": null, "allOf": [ { "$ref": "#/components/schemas/UserSpecifiedImportExportPolicy" @@ -5078,6 +5083,7 @@ }, "allowed_import": { "description": "Apply import policy to this peer with an allow list.", + "default": null, "allOf": [ { "$ref": "#/components/schemas/UserSpecifiedImportExportPolicy" @@ -5093,6 +5099,7 @@ "auth_key_id": { "nullable": true, "description": "The key identifier for authentication to use with the peer.", + "default": null, "allOf": [ { "$ref": "#/components/schemas/BgpAuthKeyId" @@ -5152,6 +5159,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -5159,6 +5167,7 @@ "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -5166,6 +5175,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -5177,6 +5187,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -5184,6 +5195,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/oximeter/collector/src/self_stats.rs b/oximeter/collector/src/self_stats.rs index ab9e5bedf4..66be514523 100644 --- a/oximeter/collector/src/self_stats.rs +++ b/oximeter/collector/src/self_stats.rs @@ -234,8 +234,11 @@ mod tests { let collections = collections(); let failed = failed_collections(); let mut set = SchemaSet::default(); - assert!(set.insert_checked(&collector, &collections).is_none()); - assert!(set.insert_checked(&collector, &failed).is_none()); + assert!(set + .insert_checked(&collector, &collections) + .unwrap() + .is_none()); + assert!(set.insert_checked(&collector, &failed).unwrap().is_none()); const PATH: &'static str = concat!( env!("CARGO_MANIFEST_DIR"), diff --git a/oximeter/collector/tests/output/self-stat-schema.json b/oximeter/collector/tests/output/self-stat-schema.json index f5e439d40f..00363cb73c 100644 --- a/oximeter/collector/tests/output/self-stat-schema.json +++ b/oximeter/collector/tests/output/self-stat-schema.json @@ -1,91 +1,120 @@ { "oximeter_collector:collections": { "timeseries_name": "oximeter_collector:collections", + "description": { + "target": "", + "metric": "" + }, "field_schema": [ { "name": "base_route", "field_type": "string", - "source": "metric" + "source": "metric", + "description": "" }, { "name": "collector_id", "field_type": "uuid", - "source": "target" + "source": "target", + "description": "" }, { "name": "collector_ip", "field_type": "ip_addr", - "source": "target" + "source": "target", + "description": "" }, { "name": "collector_port", "field_type": "u16", - "source": "target" + "source": "target", + "description": "" }, { "name": "producer_id", "field_type": "uuid", - "source": "metric" + "source": "metric", + "description": "" }, { "name": "producer_ip", "field_type": "ip_addr", - "source": "metric" + "source": "metric", + "description": "" }, { "name": "producer_port", "field_type": "u16", - "source": "metric" + "source": "metric", + "description": "" } ], "datum_type": "cumulative_u64", - "created": "2024-06-11T21:04:40.129429782Z" + "version": 1, + "authz_scope": "fleet", + "units": "count", + "created": "2024-06-19T18:42:39.343957406Z" }, "oximeter_collector:failed_collections": { "timeseries_name": "oximeter_collector:failed_collections", + "description": { + "target": "", + "metric": "" + }, "field_schema": [ { "name": "base_route", "field_type": "string", - "source": "metric" + "source": "metric", + "description": "" }, { "name": "collector_id", "field_type": "uuid", - "source": "target" + "source": "target", + "description": "" }, { "name": "collector_ip", "field_type": "ip_addr", - "source": "target" + "source": "target", + "description": "" }, { "name": "collector_port", "field_type": "u16", - "source": "target" + "source": "target", + "description": "" }, { "name": "producer_id", "field_type": "uuid", - "source": "metric" + "source": "metric", + "description": "" }, { "name": "producer_ip", "field_type": "ip_addr", - "source": "metric" + "source": "metric", + "description": "" }, { "name": "producer_port", "field_type": "u16", - "source": "metric" + "source": "metric", + "description": "" }, { "name": "reason", "field_type": "string", - "source": "metric" + "source": "metric", + "description": "" } ], "datum_type": "cumulative_u64", - "created": "2024-06-11T21:04:40.130239084Z" + "version": 1, + "authz_scope": "fleet", + "units": "count", + "created": "2024-06-19T18:42:39.344007227Z" } -} \ No newline at end of file +} diff --git a/oximeter/db/src/lib.rs b/oximeter/db/src/lib.rs index e1570ee0c3..c471a837ea 100644 --- a/oximeter/db/src/lib.rs +++ b/oximeter/db/src/lib.rs @@ -160,6 +160,12 @@ impl From for TimeseriesSchema { schema.timeseries_name.as_str(), ) .expect("Invalid timeseries name in database"), + // TODO-cleanup: Fill these in from the values in the database. See + // https://github.com/oxidecomputer/omicron/issues/5942. + description: Default::default(), + version: oximeter::schema::default_schema_version(), + authz_scope: oximeter::schema::AuthzScope::Fleet, + units: oximeter::schema::Units::Count, field_schema: schema.field_schema.into(), datum_type: schema.datum_type.into(), created: schema.created, @@ -236,9 +242,14 @@ pub struct TimeseriesPageSelector { pub(crate) type TimeseriesKey = u64; +// TODO-cleanup: Add the timeseries version in to the computation of the key. +// This will require a full drop of the database, since we're changing the +// sorting key and the timeseries key on each past sample. See +// https://github.com/oxidecomputer/omicron/issues/5942 for more details. pub(crate) fn timeseries_key(sample: &Sample) -> TimeseriesKey { timeseries_key_for( &sample.timeseries_name, + // sample.timeseries_version sample.sorted_target_fields(), sample.sorted_metric_fields(), sample.measurement.datum_type(), @@ -389,11 +400,13 @@ mod tests { name: String::from("later"), field_type: FieldType::U64, source: FieldSource::Target, + description: String::new(), }; let metric_field = FieldSchema { name: String::from("earlier"), field_type: FieldType::U64, source: FieldSource::Metric, + description: String::new(), }; let timeseries_name: TimeseriesName = "foo:bar".parse().unwrap(); let datum_type = DatumType::U64; @@ -401,6 +414,10 @@ mod tests { [target_field.clone(), metric_field.clone()].into_iter().collect(); let expected_schema = TimeseriesSchema { timeseries_name: timeseries_name.clone(), + description: Default::default(), + version: oximeter::schema::default_schema_version(), + authz_scope: oximeter::schema::AuthzScope::Fleet, + units: oximeter::schema::Units::Count, field_schema, datum_type, created: Utc::now(), diff --git a/oximeter/db/src/model.rs b/oximeter/db/src/model.rs index e7f9f56b63..810463d250 100644 --- a/oximeter/db/src/model.rs +++ b/oximeter/db/src/model.rs @@ -109,6 +109,10 @@ pub(crate) struct DbFieldList { pub types: Vec, #[serde(rename = "fields.source")] pub sources: Vec, + // TODO-completeness: Populate the description from the database here. See + // https://github.com/oxidecomputer/omicron/issues/5942 for more details. + //#[serde(rename = "fields.description")] + //pub descriptions: Vec, } impl From for BTreeSet { @@ -121,6 +125,7 @@ impl From for BTreeSet { name, field_type: ty.into(), source: source.into(), + description: String::new(), }) .collect() } @@ -149,6 +154,9 @@ pub(crate) struct DbTimeseriesSchema { pub datum_type: DbDatumType, #[serde(with = "serde_timestamp")] pub created: DateTime, + // TODO-completeness: Add the authorization scope, version, and units once + // they are tracked in the database. See + // https://github.com/oxidecomputer/omicron/issues/5942 for more details. } impl From for DbTimeseriesSchema { @@ -613,7 +621,7 @@ declare_histogram_measurement_row! { HistogramF64MeasurementRow, DbHistogram BTreeMap> { let mut out = BTreeMap::new(); for field in sample.fields() { - let timeseries_name = sample.timeseries_name.clone(); + let timeseries_name = sample.timeseries_name.to_string(); let timeseries_key = crate::timeseries_key(sample); let field_name = field.name.clone(); let (table_name, row_string) = match &field.value { @@ -761,7 +769,11 @@ pub(crate) fn unroll_measurement_row(sample: &Sample) -> (String, String) { let timeseries_name = sample.timeseries_name.clone(); let timeseries_key = crate::timeseries_key(sample); let measurement = &sample.measurement; - unroll_measurement_row_impl(timeseries_name, timeseries_key, measurement) + unroll_measurement_row_impl( + timeseries_name.to_string(), + timeseries_key, + measurement, + ) } /// Given a sample's measurement, return a table name and row to insert. @@ -1930,11 +1942,13 @@ mod tests { name: String::from("field0"), field_type: FieldType::I64, source: FieldSource::Target, + description: String::new(), }, FieldSchema { name: String::from("field1"), field_type: FieldType::IpAddr, source: FieldSource::Metric, + description: String::new(), }, ] .into_iter() @@ -1975,7 +1989,7 @@ mod tests { assert_eq!(out["oximeter.fields_i64"].len(), 1); let unpacked: StringFieldRow = serde_json::from_str(&out["oximeter.fields_string"][0]).unwrap(); - assert_eq!(unpacked.timeseries_name, sample.timeseries_name); + assert_eq!(sample.timeseries_name, unpacked.timeseries_name); let field = sample.target_fields().next().unwrap(); assert_eq!(unpacked.field_name, field.name); if let FieldValue::String(v) = &field.value { diff --git a/oximeter/db/src/query.rs b/oximeter/db/src/query.rs index e14dfbbc55..7b622920ff 100644 --- a/oximeter/db/src/query.rs +++ b/oximeter/db/src/query.rs @@ -249,7 +249,7 @@ impl SelectQueryBuilder { T: Target, M: Metric, { - let schema = TimeseriesSchema::new(target, metric); + let schema = TimeseriesSchema::new(target, metric)?; let mut builder = Self::new(&schema); let target_fields = target.field_names().iter().zip(target.field_values()); @@ -777,16 +777,22 @@ mod tests { fn test_select_query_builder_filter_raw() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), + description: Default::default(), + version: oximeter::schema::default_schema_version(), + authz_scope: oximeter::schema::AuthzScope::Fleet, + units: oximeter::schema::Units::Count, field_schema: [ FieldSchema { name: "f0".to_string(), field_type: FieldType::I64, source: FieldSource::Target, + description: String::new(), }, FieldSchema { name: "f1".to_string(), field_type: FieldType::Bool, source: FieldSource::Target, + description: String::new(), }, ] .into_iter() @@ -910,6 +916,10 @@ mod tests { fn test_select_query_builder_no_fields() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), + description: Default::default(), + version: oximeter::schema::default_schema_version(), + authz_scope: oximeter::schema::AuthzScope::Fleet, + units: oximeter::schema::Units::Count, field_schema: BTreeSet::new(), datum_type: DatumType::I64, created: Utc::now(), @@ -932,6 +942,10 @@ mod tests { fn test_select_query_builder_limit_offset() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), + description: Default::default(), + version: oximeter::schema::default_schema_version(), + authz_scope: oximeter::schema::AuthzScope::Fleet, + units: oximeter::schema::Units::Count, field_schema: BTreeSet::new(), datum_type: DatumType::I64, created: Utc::now(), @@ -1002,16 +1016,22 @@ mod tests { fn test_select_query_builder_no_selectors() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), + description: Default::default(), + version: oximeter::schema::default_schema_version(), + authz_scope: oximeter::schema::AuthzScope::Fleet, + units: oximeter::schema::Units::Count, field_schema: [ FieldSchema { name: "f0".to_string(), field_type: FieldType::I64, source: FieldSource::Target, + description: String::new(), }, FieldSchema { name: "f1".to_string(), field_type: FieldType::Bool, source: FieldSource::Target, + description: String::new(), }, ] .into_iter() @@ -1065,16 +1085,22 @@ mod tests { fn test_select_query_builder_field_selectors() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), + description: Default::default(), + version: oximeter::schema::default_schema_version(), + authz_scope: oximeter::schema::AuthzScope::Fleet, + units: oximeter::schema::Units::Count, field_schema: [ FieldSchema { name: "f0".to_string(), field_type: FieldType::I64, source: FieldSource::Target, + description: String::new(), }, FieldSchema { name: "f1".to_string(), field_type: FieldType::Bool, source: FieldSource::Target, + description: String::new(), }, ] .into_iter() @@ -1116,16 +1142,22 @@ mod tests { fn test_select_query_builder_full() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), + description: Default::default(), + version: oximeter::schema::default_schema_version(), + authz_scope: oximeter::schema::AuthzScope::Fleet, + units: oximeter::schema::Units::Count, field_schema: [ FieldSchema { name: "f0".to_string(), field_type: FieldType::I64, source: FieldSource::Target, + description: String::new(), }, FieldSchema { name: "f1".to_string(), field_type: FieldType::Bool, source: FieldSource::Target, + description: String::new(), }, ] .into_iter() diff --git a/oximeter/impl/Cargo.toml b/oximeter/impl/Cargo.toml new file mode 100644 index 0000000000..a8b42d41cd --- /dev/null +++ b/oximeter/impl/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "oximeter-impl" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[lints] +workspace = true + +[dependencies] +bytes = { workspace = true, features = [ "serde" ] } +chrono.workspace = true +float-ord.workspace = true +heck.workspace = true +num.workspace = true +omicron-common.workspace = true +omicron-workspace-hack.workspace = true +oximeter-macro-impl.workspace = true +prettyplease.workspace = true +proc-macro2.workspace = true +quote.workspace = true +regex.workspace = true +schemars = { workspace = true, features = [ "uuid1", "bytes", "chrono" ] } +serde.workspace = true +serde_json.workspace = true +slog-error-chain.workspace = true +strum.workspace = true +syn.workspace = true +toml.workspace = true +thiserror.workspace = true +uuid.workspace = true + +[dev-dependencies] +approx.workspace = true +rand = { workspace = true, features = ["std_rng"] } +rand_distr.workspace = true +rstest.workspace = true +serde_json.workspace = true +trybuild.workspace = true diff --git a/oximeter/oximeter/src/histogram.rs b/oximeter/impl/src/histogram.rs similarity index 99% rename from oximeter/oximeter/src/histogram.rs rename to oximeter/impl/src/histogram.rs index 9ce7b65121..0fb175555e 100644 --- a/oximeter/oximeter/src/histogram.rs +++ b/oximeter/impl/src/histogram.rs @@ -509,6 +509,9 @@ where /// Example /// ------- /// ```rust + /// # // Rename the impl crate so the doctests can refer to the public + /// # // `oximeter` crate, not the private impl. + /// # use oximeter_impl as oximeter; /// use oximeter::histogram::Histogram; /// /// let hist = Histogram::with_bins(&[(0..10).into(), (10..100).into()]).unwrap(); @@ -905,6 +908,9 @@ where /// ------- /// /// ```rust + /// # // Rename the impl crate so the doctests can refer to the public + /// # // `oximeter` crate, not the private impl. + /// # use oximeter_impl as oximeter; /// use oximeter::histogram::{Histogram, BinRange}; /// use std::ops::{RangeBounds, Bound}; /// diff --git a/oximeter/impl/src/lib.rs b/oximeter/impl/src/lib.rs new file mode 100644 index 0000000000..5acbeb9422 --- /dev/null +++ b/oximeter/impl/src/lib.rs @@ -0,0 +1,51 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2024 Oxide Computer Company + +pub use oximeter_macro_impl::*; + +// Export the current crate as `oximeter`. The macros defined in `oximeter-macro-impl` generate +// code referring to symbols like `oximeter::traits::Target`. In consumers of this crate, that's +// fine, but internally there _is_ no crate named `oximeter`, it's just `self` or `crate`. +// +// See https://github.com/rust-lang/rust/pull/55275 for the PR introducing this fix, which links to +// lots of related issues and discussion. +extern crate self as oximeter; + +pub mod histogram; +pub mod quantile; +pub mod schema; +pub mod test_util; +pub mod traits; +pub mod types; + +pub use quantile::Quantile; +pub use quantile::QuantileError; +pub use schema::FieldSchema; +pub use schema::TimeseriesName; +pub use schema::TimeseriesSchema; +pub use traits::Metric; +pub use traits::Producer; +pub use traits::Target; +pub use types::Datum; +pub use types::DatumType; +pub use types::Field; +pub use types::FieldType; +pub use types::FieldValue; +pub use types::Measurement; +pub use types::MetricsError; +pub use types::Sample; + +/// Construct the timeseries name for a Target and Metric. +pub fn timeseries_name( + target: &T, + metric: &M, +) -> Result +where + T: Target, + M: Metric, +{ + TimeseriesName::try_from(format!("{}:{}", target.name(), metric.name())) +} diff --git a/oximeter/oximeter/src/quantile.rs b/oximeter/impl/src/quantile.rs similarity index 96% rename from oximeter/oximeter/src/quantile.rs rename to oximeter/impl/src/quantile.rs index 8bc144bb0a..3e070cc302 100644 --- a/oximeter/oximeter/src/quantile.rs +++ b/oximeter/impl/src/quantile.rs @@ -78,6 +78,9 @@ impl Quantile { /// # Examples /// /// ``` + /// # // Rename the impl crate so the doctests can refer to the public + /// # // `oximeter` crate, not the private impl. + /// # use oximeter_impl as oximeter; /// use oximeter::Quantile; /// let q = Quantile::new(0.5).unwrap(); /// @@ -113,6 +116,9 @@ impl Quantile { /// /// # Examples /// ``` + /// # // Rename the impl crate so the doctests can refer to the public + /// # // `oximeter` crate, not the private impl. + /// # use oximeter_impl as oximeter; /// use oximeter::Quantile; /// let q = Quantile::from_parts( /// 0.5, @@ -194,6 +200,9 @@ impl Quantile { /// # Examples /// /// ``` + /// # // Rename the impl crate so the doctests can refer to the public + /// # // `oximeter` crate, not the private impl. + /// # use oximeter_impl as oximeter; /// use oximeter::Quantile; /// let mut q = Quantile::new(0.5).unwrap(); /// for o in 1..=100 { @@ -234,6 +243,9 @@ impl Quantile { /// # Examples /// /// ``` + /// # // Rename the impl crate so the doctests can refer to the public + /// # // `oximeter` crate, not the private impl. + /// # use oximeter_impl as oximeter; /// use oximeter::Quantile; /// let mut q = Quantile::new(0.9).unwrap(); /// q.append(10).unwrap(); diff --git a/oximeter/impl/src/schema/codegen.rs b/oximeter/impl/src/schema/codegen.rs new file mode 100644 index 0000000000..4aa09cf136 --- /dev/null +++ b/oximeter/impl/src/schema/codegen.rs @@ -0,0 +1,491 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2024 Oxide Computer Company + +//! Generate Rust types and code from oximeter schema definitions. + +use crate::schema::ir::find_schema_version; +use crate::schema::ir::load_schema; +use crate::schema::AuthzScope; +use crate::schema::FieldSource; +use crate::schema::Units; +use crate::DatumType; +use crate::FieldSchema; +use crate::FieldType; +use crate::MetricsError; +use crate::TimeseriesSchema; +use chrono::prelude::DateTime; +use chrono::prelude::Utc; +use proc_macro2::TokenStream; +use quote::quote; + +/// Emit types for using one timeseries definition. +/// +/// Provided with a TOML-formatted schema definition, this emits Rust types for +/// the target and metric from the latest version; and a function that returns +/// the `TimeseriesSchema` for _all_ versions of the timeseries. +/// +/// Both of these items are emitted in a module with the same name as the +/// target. +pub fn use_timeseries(contents: &str) -> Result { + let schema = load_schema(contents)?; + let latest = find_schema_version(schema.iter().cloned(), None); + let mod_name = quote::format_ident!("{}", latest[0].target_name()); + let types = emit_schema_types(latest); + let func = emit_schema_function(schema.into_iter()); + Ok(quote! { + pub mod #mod_name { + #types + #func + } + }) +} + +fn emit_schema_function( + list: impl Iterator, +) -> TokenStream { + quote! { + pub fn timeseries_schema() -> Vec<::oximeter::schema::TimeseriesSchema> { + vec![ + #(#list),* + ] + } + } +} + +fn emit_schema_types(list: Vec) -> TokenStream { + let first_schema = list.first().expect("load_schema ensures non-empty"); + let target_def = emit_target(first_schema); + let metric_defs = emit_metrics(&list); + quote! { + #target_def + #metric_defs + } +} + +fn emit_metrics(schema: &[TimeseriesSchema]) -> TokenStream { + let items = schema.iter().map(|s| emit_one(FieldSource::Metric, s)); + quote! { #(#items)* } +} + +fn emit_target(schema: &TimeseriesSchema) -> TokenStream { + emit_one(FieldSource::Target, schema) +} + +fn emit_one(source: FieldSource, schema: &TimeseriesSchema) -> TokenStream { + let name = match source { + FieldSource::Target => schema.target_name(), + FieldSource::Metric => schema.metric_name(), + }; + let struct_name = + quote::format_ident!("{}", format!("{}", heck::AsPascalCase(name))); + let field_defs: Vec<_> = schema + .field_schema + .iter() + .filter_map(|s| { + if s.source == source { + let name = quote::format_ident!("{}", s.name); + let type_ = emit_rust_type_for_field(s.field_type); + let docstring = s.description.as_str(); + Some(quote! { + #[doc = #docstring] + pub #name: #type_ + }) + } else { + None + } + }) + .collect(); + let (oximeter_trait, maybe_datum, type_docstring) = match source { + FieldSource::Target => ( + quote! {::oximeter::Target }, + quote! {}, + schema.description.target.as_str(), + ), + FieldSource::Metric => { + let datum_type = emit_rust_type_for_datum_type(schema.datum_type); + ( + quote! { ::oximeter::Metric }, + quote! { pub datum: #datum_type, }, + schema.description.metric.as_str(), + ) + } + }; + quote! { + #[doc = #type_docstring] + #[derive(Clone, Debug, PartialEq, #oximeter_trait)] + pub struct #struct_name { + #( #field_defs, )* + #maybe_datum + } + } +} + +// Implement ToTokens for the components of a `TimeseriesSchema`. +// +// This is used so that we can emit a function that will return the same data as +// we parse from the TOML file with the timeseries definition, as a way to +// export the definitions without needing that actual file at runtime. +impl quote::ToTokens for DatumType { + fn to_tokens(&self, tokens: &mut TokenStream) { + let toks = match self { + DatumType::Bool => quote! { ::oximeter::DatumType::Bool }, + DatumType::I8 => quote! { ::oximeter::DatumType::I8 }, + DatumType::U8 => quote! { ::oximeter::DatumType::U8 }, + DatumType::I16 => quote! { ::oximeter::DatumType::I16 }, + DatumType::U16 => quote! { ::oximeter::DatumType::U16 }, + DatumType::I32 => quote! { ::oximeter::DatumType::I32 }, + DatumType::U32 => quote! { ::oximeter::DatumType::U32 }, + DatumType::I64 => quote! { ::oximeter::DatumType::I64 }, + DatumType::U64 => quote! { ::oximeter::DatumType::U64 }, + DatumType::F32 => quote! { ::oximeter::DatumType::F32 }, + DatumType::F64 => quote! { ::oximeter::DatumType::F64 }, + DatumType::String => quote! { ::oximeter::DatumType::String }, + DatumType::Bytes => quote! { ::oximeter::DatumType::Bytes }, + DatumType::CumulativeI64 => { + quote! { ::oximeter::DatumType::CumulativeI64 } + } + DatumType::CumulativeU64 => { + quote! { ::oximeter::DatumType::CumulativeU64 } + } + DatumType::CumulativeF32 => { + quote! { ::oximeter::DatumType::CumulativeF32 } + } + DatumType::CumulativeF64 => { + quote! { ::oximeter::DatumType::CumulativeF64 } + } + DatumType::HistogramI8 => { + quote! { ::oximeter::DatumType::HistogramI8 } + } + DatumType::HistogramU8 => { + quote! { ::oximeter::DatumType::HistogramU8 } + } + DatumType::HistogramI16 => { + quote! { ::oximeter::DatumType::HistogramI16 } + } + DatumType::HistogramU16 => { + quote! { ::oximeter::DatumType::HistogramU16 } + } + DatumType::HistogramI32 => { + quote! { ::oximeter::DatumType::HistogramI32 } + } + DatumType::HistogramU32 => { + quote! { ::oximeter::DatumType::HistogramU32 } + } + DatumType::HistogramI64 => { + quote! { ::oximeter::DatumType::HistogramI64 } + } + DatumType::HistogramU64 => { + quote! { ::oximeter::DatumType::HistogramU64 } + } + DatumType::HistogramF32 => { + quote! { ::oximeter::DatumType::HistogramF32 } + } + DatumType::HistogramF64 => { + quote! { ::oximeter::DatumType::HistogramF64 } + } + }; + toks.to_tokens(tokens); + } +} + +// Emit tokens representing the Rust path matching the provided datum type. +fn emit_rust_type_for_datum_type(datum_type: DatumType) -> TokenStream { + match datum_type { + DatumType::Bool => quote! { bool }, + DatumType::I8 => quote! { i8 }, + DatumType::U8 => quote! { u8 }, + DatumType::I16 => quote! { i16 }, + DatumType::U16 => quote! { u16 }, + DatumType::I32 => quote! { i32 }, + DatumType::U32 => quote! { u32 }, + DatumType::I64 => quote! { i64 }, + DatumType::U64 => quote! { u64 }, + DatumType::F32 => quote! { f32 }, + DatumType::F64 => quote! { f64 }, + DatumType::String => quote! { String }, + DatumType::Bytes => quote! { ::bytes::Bytes }, + DatumType::CumulativeI64 => { + quote! { ::oximeter::types::Cumulative } + } + DatumType::CumulativeU64 => { + quote! { ::oximeter::types::Cumulative } + } + DatumType::CumulativeF32 => { + quote! { ::oximeter::types::Cumulative } + } + DatumType::CumulativeF64 => { + quote! { ::oximeter::types::Cumulative } + } + DatumType::HistogramI8 => { + quote! { ::oximeter::histogram::Histogram } + } + DatumType::HistogramU8 => { + quote! { ::oximeter::histogram::Histogram } + } + DatumType::HistogramI16 => { + quote! { ::oximeter::histogram::Histogram } + } + DatumType::HistogramU16 => { + quote! { ::oximeter::histogram::Histogram } + } + DatumType::HistogramI32 => { + quote! { ::oximeter::histogram::Histogram } + } + DatumType::HistogramU32 => { + quote! { ::oximeter::histogram::Histogram } + } + DatumType::HistogramI64 => { + quote! { ::oximeter::histogram::Histogram } + } + DatumType::HistogramU64 => { + quote! { ::oximeter::histogram::Histogram } + } + DatumType::HistogramF32 => { + quote! { ::oximeter::histogram::Histogram } + } + DatumType::HistogramF64 => { + quote! { ::oximeter::histogram::Histogram } + } + } +} + +// Generate the quoted path to the Rust type matching the given field type. +fn emit_rust_type_for_field(field_type: FieldType) -> TokenStream { + match field_type { + FieldType::String => quote! { ::std::borrow::Cow<'static, str> }, + FieldType::I8 => quote! { i8 }, + FieldType::U8 => quote! { u8 }, + FieldType::I16 => quote! { i16 }, + FieldType::U16 => quote! { u16 }, + FieldType::I32 => quote! { i32 }, + FieldType::U32 => quote! { u32 }, + FieldType::I64 => quote! { i64 }, + FieldType::U64 => quote! { u64 }, + FieldType::IpAddr => quote! { ::core::net::IpAddr }, + FieldType::Uuid => quote! { ::uuid::Uuid }, + FieldType::Bool => quote! { bool }, + } +} + +impl quote::ToTokens for FieldSource { + fn to_tokens(&self, tokens: &mut TokenStream) { + let toks = match self { + FieldSource::Target => { + quote! { ::oximeter::schema::FieldSource::Target } + } + FieldSource::Metric => { + quote! { ::oximeter::schema::FieldSource::Metric } + } + }; + toks.to_tokens(tokens); + } +} + +impl quote::ToTokens for FieldType { + fn to_tokens(&self, tokens: &mut TokenStream) { + let toks = match self { + FieldType::String => quote! { ::oximeter::FieldType::String }, + FieldType::I8 => quote! { ::oximeter::FieldType::I8 }, + FieldType::U8 => quote! { ::oximeter::FieldType::U8 }, + FieldType::I16 => quote! { ::oximeter::FieldType::I16 }, + FieldType::U16 => quote! { ::oximeter::FieldType::U16 }, + FieldType::I32 => quote! { ::oximeter::FieldType::I32 }, + FieldType::U32 => quote! { ::oximeter::FieldType::U32 }, + FieldType::I64 => quote! { ::oximeter::FieldType::I64 }, + FieldType::U64 => quote! { ::oximeter::FieldType::U64 }, + FieldType::IpAddr => quote! { ::oximeter::FieldType::IpAddr }, + FieldType::Uuid => quote! { ::oximeter::FieldType::Uuid }, + FieldType::Bool => quote! { ::oximeter::FieldType::Bool }, + }; + toks.to_tokens(tokens); + } +} + +impl quote::ToTokens for AuthzScope { + fn to_tokens(&self, tokens: &mut TokenStream) { + let toks = match self { + AuthzScope::Fleet => { + quote! { ::oximeter::schema::AuthzScope::Fleet } + } + AuthzScope::Silo => quote! { ::oximeter::schema::AuthzScope::Silo }, + AuthzScope::Project => { + quote! { ::oximeter::schema::AuthzScope::Project } + } + AuthzScope::ViewableToAll => { + quote! { ::oximeter::schema::AuthzScope::ViewableToAll } + } + }; + toks.to_tokens(tokens); + } +} + +fn quote_creation_time(created: DateTime) -> TokenStream { + let secs = created.timestamp(); + let nsecs = created.timestamp_subsec_nanos(); + quote! { + ::chrono::DateTime::from_timestamp(#secs, #nsecs).unwrap() + } +} + +impl quote::ToTokens for Units { + fn to_tokens(&self, tokens: &mut TokenStream) { + let toks = match self { + Units::Count => quote! { ::oximeter::schema::Units::Count }, + Units::Bytes => quote! { ::oximeter::schema::Units::Bytes }, + }; + toks.to_tokens(tokens); + } +} + +impl quote::ToTokens for FieldSchema { + fn to_tokens(&self, tokens: &mut TokenStream) { + let name = self.name.as_str(); + let field_type = self.field_type; + let source = self.source; + let description = self.description.as_str(); + let toks = quote! { + ::oximeter::FieldSchema { + name: String::from(#name), + field_type: #field_type, + source: #source, + description: String::from(#description), + } + }; + toks.to_tokens(tokens); + } +} + +impl quote::ToTokens for TimeseriesSchema { + fn to_tokens(&self, tokens: &mut TokenStream) { + let field_schema = &self.field_schema; + let timeseries_name = self.timeseries_name.to_string(); + let target_description = self.description.target.as_str(); + let metric_description = self.description.metric.as_str(); + let authz_scope = self.authz_scope; + let units = self.units; + let datum_type = self.datum_type; + let ver = self.version.get(); + let version = quote! { ::core::num::NonZeroU8::new(#ver).unwrap() }; + let created = quote_creation_time(self.created); + let toks = quote! { + ::oximeter::schema::TimeseriesSchema { + timeseries_name: ::oximeter::TimeseriesName::try_from(#timeseries_name).unwrap(), + description: ::oximeter::schema::TimeseriesDescription { + target: String::from(#target_description), + metric: String::from(#metric_description), + }, + authz_scope: #authz_scope, + units: #units, + field_schema: ::std::collections::BTreeSet::from([ + #(#field_schema),* + ]), + datum_type: #datum_type, + version: #version, + created: #created, + } + }; + toks.to_tokens(tokens); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::schema::TimeseriesDescription; + use std::{collections::BTreeSet, num::NonZeroU8}; + + #[test] + fn emit_schema_types_generates_expected_tokens() { + let schema = TimeseriesSchema { + timeseries_name: "foo:bar".parse().unwrap(), + description: TimeseriesDescription { + target: "a target".into(), + metric: "a metric".into(), + }, + field_schema: BTreeSet::from([ + FieldSchema { + name: "f0".into(), + field_type: FieldType::String, + source: FieldSource::Target, + description: "target field".into(), + }, + FieldSchema { + name: "f1".into(), + field_type: FieldType::Uuid, + source: FieldSource::Metric, + description: "metric field".into(), + }, + ]), + datum_type: DatumType::CumulativeU64, + version: NonZeroU8::new(1).unwrap(), + authz_scope: AuthzScope::Fleet, + units: Units::Bytes, + created: Utc::now(), + }; + + let tokens = emit_schema_types(vec![schema.clone()]); + + let expected = quote! { + #[doc = "a target"] + #[derive(Clone, Debug, PartialEq, ::oximeter::Target)] + pub struct Foo { + #[doc = "target field"] + pub f0: ::std::borrow::Cow<'static, str>, + } + + #[doc = "a metric"] + #[derive(Clone, Debug, PartialEq, ::oximeter::Metric)] + pub struct Bar { + #[doc = "metric field"] + pub f1: ::uuid::Uuid, + pub datum: ::oximeter::types::Cumulative, + } + }; + + assert_eq!(tokens.to_string(), expected.to_string()); + } + + #[test] + fn emit_schema_types_with_no_metric_fields_generates_expected_tokens() { + let schema = TimeseriesSchema { + timeseries_name: "foo:bar".parse().unwrap(), + description: TimeseriesDescription { + target: "a target".into(), + metric: "a metric".into(), + }, + field_schema: BTreeSet::from([FieldSchema { + name: "f0".into(), + field_type: FieldType::String, + source: FieldSource::Target, + description: "target field".into(), + }]), + datum_type: DatumType::CumulativeU64, + version: NonZeroU8::new(1).unwrap(), + authz_scope: AuthzScope::Fleet, + units: Units::Bytes, + created: Utc::now(), + }; + + let tokens = emit_schema_types(vec![schema.clone()]); + + let expected = quote! { + #[doc = "a target"] + #[derive(Clone, Debug, PartialEq, ::oximeter::Target)] + pub struct Foo { + #[doc = "target field"] + pub f0: ::std::borrow::Cow<'static, str>, + } + + #[doc = "a metric"] + #[derive(Clone, Debug, PartialEq, ::oximeter::Metric)] + pub struct Bar { + pub datum: ::oximeter::types::Cumulative, + } + }; + + assert_eq!(tokens.to_string(), expected.to_string()); + } +} diff --git a/oximeter/impl/src/schema/ir.rs b/oximeter/impl/src/schema/ir.rs new file mode 100644 index 0000000000..573af9c2b0 --- /dev/null +++ b/oximeter/impl/src/schema/ir.rs @@ -0,0 +1,1416 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2024 Oxide Computer Company + +//! Serialization of timeseries schema definitions. +//! +//! These types are used as an intermediate representation of schema. The schema +//! are written in TOML; deserialized into these types; and then either +//! inspected or used to generate code that contains the equivalent Rust types +//! and trait implementations. + +use crate::schema::AuthzScope; +use crate::schema::DatumType; +use crate::schema::FieldSource; +use crate::schema::FieldType; +use crate::schema::TimeseriesDescription; +use crate::schema::Units; +use crate::FieldSchema; +use crate::MetricsError; +use crate::TimeseriesName; +use crate::TimeseriesSchema; +use chrono::Utc; +use serde::Deserialize; +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::num::NonZeroU8; + +#[derive(Debug, Deserialize)] +pub struct FieldMetadata { + #[serde(rename = "type")] + pub type_: FieldType, + pub description: String, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub enum MetricFields { + Removed { removed_in: NonZeroU8 }, + Added { added_in: NonZeroU8, fields: Vec }, + Versioned(VersionedFields), +} + +#[derive(Debug, Deserialize)] +pub struct VersionedFields { + pub version: NonZeroU8, + pub fields: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct TargetDefinition { + pub name: String, + pub description: String, + pub authz_scope: AuthzScope, + pub versions: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct MetricDefinition { + pub name: String, + pub description: String, + pub units: Units, + pub datum_type: DatumType, + pub versions: Vec, +} + +fn checked_version_deser<'de, D>(d: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let x = NonZeroU8::deserialize(d)?; + if x.get() == 1 { + Ok(x) + } else { + Err(serde::de::Error::custom(format!( + "Only version 1 of the timeseries defintion format \ + is currently supported, found version {x}", + ))) + } +} + +#[derive(Debug, Deserialize)] +pub struct TimeseriesDefinition { + #[serde(deserialize_with = "checked_version_deser")] + pub format_version: NonZeroU8, + pub target: TargetDefinition, + pub metrics: Vec, + pub fields: BTreeMap, +} + +impl TimeseriesDefinition { + pub fn into_schema_list( + self, + ) -> Result, MetricsError> { + if self.target.versions.is_empty() { + return Err(MetricsError::SchemaDefinition(String::from( + "At least one target version must be defined", + ))); + } + if self.metrics.is_empty() { + return Err(MetricsError::SchemaDefinition(String::from( + "At least one metric must be defined", + ))); + } + let mut timeseries = BTreeMap::new(); + let target_name = &self.target.name; + + // At this point, we do not support actually _modifying_ schema. + // Instead, we're putting in place infrastructure to support multiple + // versions, while still requiring all schema to define the first and + // only the first version. + // + // We omit this check in tests, to ensure that the code correctly + // handles updates. + #[cfg(not(test))] + if self.target.versions.len() > 1 + || self + .target + .versions + .iter() + .any(|fields| fields.version.get() > 1) + { + return Err(MetricsError::SchemaDefinition(String::from( + "Exactly one timeseries version, with version number 1, \ + may currently be specified. Updates will be supported \ + in future releases.", + ))); + } + + // First create a map from target version to the fields in it. + // + // This is used to do O(lg n) lookups into the set of target fields when we + // iterate through metric versions below, i.e., avoiding quadratic behavior. + let mut target_fields_by_version = BTreeMap::new(); + for (expected_version, target_fields) in + (1u8..).zip(self.target.versions.iter()) + { + if expected_version != target_fields.version.get() { + return Err(MetricsError::SchemaDefinition(format!( + "Target '{}' versions should be sequential \ + and monotonically increasing (expected {}, found {})", + target_name, expected_version, target_fields.version, + ))); + } + + let fields: BTreeSet<_> = + target_fields.fields.iter().cloned().collect(); + if fields.len() != target_fields.fields.len() { + return Err(MetricsError::SchemaDefinition(format!( + "Target '{}' version {} lists duplicate field names", + target_name, expected_version, + ))); + } + if fields.is_empty() { + return Err(MetricsError::SchemaDefinition(format!( + "Target '{}' version {} must have at least one field", + target_name, expected_version, + ))); + } + + if target_fields_by_version + .insert(expected_version, fields) + .is_some() + { + return Err(MetricsError::SchemaDefinition(format!( + "Target '{}' version {} is duplicated", + target_name, expected_version, + ))); + } + } + + // Start by looping over all the metrics in the definition. + // + // As we do so, we'll attach the target definition at the corresponding + // version, along with running some basic lints and checks. + for metric in self.metrics.iter() { + let metric_name = &metric.name; + + // Store the current version of the metric. This doesn't need to be + // sequential, but they do need to be monotonic and have a matching + // target version. We'll fill in any gaps with the last active version + // of the metric (if any). + let mut current_version: Option = None; + + // Also store the last used version of the target. This lets users omit + // an unchanged metric, and we use this value to fill in the implied + // version of the metric. + let mut last_target_version: u8 = 0; + + // Iterate through each version of this metric. + // + // In general, we expect metrics to be addded in the first version; + // modified by adding / removing fields; and possibly removed at the + // end. However, they can be added / removed multiple times, and not + // added until a later version of the target. + for metric_fields in metric.versions.iter() { + // Fill in any gaps from the last target version to this next + // metric version. This only works once we've filled in at least + // one version of the metric, and stored the current version / + // fields. + if let Some(current) = current_version.as_ref() { + let current_fields = current + .fields() + .expect("Should have some fields if we have any previous version"); + while last_target_version <= current.version().get() { + last_target_version += 1; + let Some(target_fields) = + target_fields_by_version.get(&last_target_version) + else { + return Err(MetricsError::SchemaDefinition( + format!( + "Metric '{}' version {} does not have \ + a matching version in the target '{}'", + metric_name, + last_target_version, + target_name, + ), + )); + }; + let field_schema = construct_field_schema( + &self.fields, + target_name, + target_fields, + metric_name, + current_fields, + )?; + let authz_scope = extract_authz_scope( + metric_name, + self.target.authz_scope, + &field_schema, + )?; + let timeseries_name = TimeseriesName::try_from( + format!("{}:{}", target_name, metric_name), + )?; + let version = + NonZeroU8::new(last_target_version).unwrap(); + let description = TimeseriesDescription { + target: self.target.description.clone(), + metric: metric.description.clone(), + }; + let schema = TimeseriesSchema { + timeseries_name: timeseries_name.clone(), + description, + field_schema, + datum_type: metric.datum_type, + version, + authz_scope, + units: metric.units, + created: Utc::now(), + }; + if let Some(old) = timeseries + .insert((timeseries_name, version), schema) + { + return Err(MetricsError::SchemaDefinition( + format!( + "Timeseries '{}' version {} is duplicated", + old.timeseries_name, old.version, + ), + )); + } + } + } + + // Extract the fields named in this version, checking that they're + // compatible with the last known version, if any. + let new_version = extract_metric_fields( + metric_name, + metric_fields, + ¤t_version, + )?; + let version = current_version.insert(new_version); + let Some(metric_fields) = version.fields() else { + continue; + }; + + // Now, insert the _next_ version of the metric with the + // validated fields we've collected for it. + last_target_version += 1; + let Some(target_fields) = + target_fields_by_version.get(&last_target_version) + else { + return Err(MetricsError::SchemaDefinition(format!( + "Metric '{}' version {} does not have \ + a matching version in the target '{}'", + metric_name, last_target_version, target_name, + ))); + }; + let field_schema = construct_field_schema( + &self.fields, + target_name, + target_fields, + metric_name, + metric_fields, + )?; + let authz_scope = extract_authz_scope( + metric_name, + self.target.authz_scope, + &field_schema, + )?; + let timeseries_name = TimeseriesName::try_from(format!( + "{}:{}", + target_name, metric_name + ))?; + let version = NonZeroU8::new(last_target_version).unwrap(); + let description = TimeseriesDescription { + target: self.target.description.clone(), + metric: metric.description.clone(), + }; + let schema = TimeseriesSchema { + timeseries_name: timeseries_name.clone(), + description, + field_schema, + datum_type: metric.datum_type, + version, + authz_scope, + units: metric.units, + created: Utc::now(), + }; + if let Some(old) = + timeseries.insert((timeseries_name, version), schema) + { + return Err(MetricsError::SchemaDefinition(format!( + "Timeseries '{}' version {} is duplicated", + old.timeseries_name, old.version, + ))); + } + } + + // We also allow omitting later versions of metrics if they are + // unchanged. A target has to specify every version, even if it's the + // same, but the metrics need only specify differences. + // + // Here, look for any target version strictly later than the last metric + // version, and create a corresponding target / metric pair for it. + if let Some(last_metric_fields) = metric.versions.last() { + match last_metric_fields { + MetricFields::Removed { .. } => {} + MetricFields::Added { + added_in: last_metric_version, + fields, + } + | MetricFields::Versioned(VersionedFields { + version: last_metric_version, + fields, + }) => { + let metric_field_names: BTreeSet<_> = + fields.iter().cloned().collect(); + let next_version = last_metric_version + .get() + .checked_add(1) + .expect("version < 256"); + for (version, target_fields) in + target_fields_by_version.range(next_version..) + { + let field_schema = construct_field_schema( + &self.fields, + target_name, + target_fields, + metric_name, + &metric_field_names, + )?; + let authz_scope = extract_authz_scope( + metric_name, + self.target.authz_scope, + &field_schema, + )?; + let timeseries_name = TimeseriesName::try_from( + format!("{}:{}", target_name, metric_name), + )?; + let version = NonZeroU8::new(*version).unwrap(); + let description = TimeseriesDescription { + target: self.target.description.clone(), + metric: metric.description.clone(), + }; + let schema = TimeseriesSchema { + timeseries_name: timeseries_name.clone(), + description, + field_schema, + datum_type: metric.datum_type, + version, + authz_scope, + units: metric.units, + created: Utc::now(), + }; + if let Some(old) = timeseries + .insert((timeseries_name, version), schema) + { + return Err(MetricsError::SchemaDefinition( + format!( + "Timeseries '{}' version {} is duplicated", + old.timeseries_name, old.version, + ), + )); + } + } + } + } + } + } + Ok(timeseries.into_values().collect()) + } +} + +#[derive(Clone, Debug)] +enum CurrentVersion { + Active { version: NonZeroU8, fields: BTreeSet }, + Inactive { removed_in: NonZeroU8 }, +} + +impl CurrentVersion { + fn version(&self) -> NonZeroU8 { + match self { + CurrentVersion::Active { version, .. } => *version, + CurrentVersion::Inactive { removed_in } => *removed_in, + } + } + + fn fields(&self) -> Option<&BTreeSet> { + match self { + CurrentVersion::Active { fields, .. } => Some(fields), + CurrentVersion::Inactive { .. } => None, + } + } +} + +/// Load the list of timeseries schema from a schema definition in TOML format. +pub fn load_schema( + contents: &str, +) -> Result, MetricsError> { + toml::from_str::(contents) + .map_err(|e| { + MetricsError::Toml( + slog_error_chain::InlineErrorChain::new(&e).to_string(), + ) + }) + .and_then(TimeseriesDefinition::into_schema_list) +} + +// Find schema of a specified version in an iterator, or the latest. +pub(super) fn find_schema_version( + list: impl Iterator, + version: Option, +) -> Vec { + match version { + Some(ver) => list.into_iter().filter(|s| s.version == ver).collect(), + None => { + let mut last_version = BTreeMap::new(); + for schema in list { + let metric_name = schema.metric_name().to_string(); + match last_version.entry(metric_name) { + Entry::Vacant(entry) => { + entry.insert((schema.version, schema.clone())); + } + Entry::Occupied(mut entry) => { + let existing_version = entry.get().0; + if existing_version < schema.version { + entry.insert((schema.version, schema.clone())); + } + } + } + } + last_version.into_values().map(|(_ver, schema)| schema).collect() + } + } +} + +fn extract_authz_scope( + metric_name: &str, + authz_scope: AuthzScope, + field_schema: &BTreeSet, +) -> Result { + let check_for_key = |scope: &str| { + let key = format!("{scope}_id"); + if field_schema.iter().any(|field| { + field.name == key && field.field_type == FieldType::Uuid + }) { + Ok(()) + } else { + Err(MetricsError::SchemaDefinition(format!( + "Metric '{}' has '{}' authorization scope, and so must \ + contain a field '{}' of UUID type", + metric_name, scope, key, + ))) + } + }; + match authz_scope { + AuthzScope::Silo => check_for_key("silo")?, + AuthzScope::Project => check_for_key("project")?, + AuthzScope::Fleet | AuthzScope::ViewableToAll => {} + } + Ok(authz_scope) +} + +fn construct_field_schema( + all_fields: &BTreeMap, + target_name: &str, + target_fields: &BTreeSet, + metric_name: &str, + metric_field_names: &BTreeSet, +) -> Result, MetricsError> { + if let Some(dup) = target_fields.intersection(&metric_field_names).next() { + return Err(MetricsError::SchemaDefinition(format!( + "Field '{}' is duplicated between target \ + '{}' and metric '{}'", + dup, target_name, metric_name, + ))); + } + + let mut field_schema = BTreeSet::new(); + for (field_name, source) in + target_fields.iter().zip(std::iter::repeat(FieldSource::Target)).chain( + metric_field_names + .iter() + .zip(std::iter::repeat(FieldSource::Metric)), + ) + { + let Some(metadata) = all_fields.get(field_name.as_str()) else { + let (kind, name) = if source == FieldSource::Target { + ("target", target_name) + } else { + ("metric", metric_name) + }; + return Err(MetricsError::SchemaDefinition(format!( + "Field '{}' is referenced in the {} '{}', but it \ + does not appear in the set of all fields.", + field_name, kind, name, + ))); + }; + validate_field_name(field_name, metadata)?; + field_schema.insert(FieldSchema { + name: field_name.to_string(), + field_type: metadata.type_, + source, + description: metadata.description.clone(), + }); + } + Ok(field_schema) +} + +fn is_snake_case(s: &str) -> bool { + s == format!("{}", heck::AsSnakeCase(s)) +} + +fn is_valid_ident_name(s: &str) -> bool { + syn::parse_str::(s).is_ok() && is_snake_case(s) +} + +fn validate_field_name( + field_name: &str, + metadata: &FieldMetadata, +) -> Result<(), MetricsError> { + if !is_valid_ident_name(field_name) { + return Err(MetricsError::SchemaDefinition(format!( + "Field name '{}' should be a valid identifier in snake_case", + field_name, + ))); + } + if metadata.type_ == FieldType::Uuid + && !(field_name.ends_with("_id") || field_name == "id") + { + return Err(MetricsError::SchemaDefinition(format!( + "Uuid field '{}' should end with '_id' or equal 'id'", + field_name, + ))); + } + Ok(()) +} + +fn extract_metric_fields<'a>( + metric_name: &'a str, + metric_fields: &'a MetricFields, + current_version: &Option, +) -> Result { + let new_version = match metric_fields { + MetricFields::Removed { removed_in } => { + match current_version { + Some(CurrentVersion::Active { version, .. }) => { + // This metric was active, and is now being + // removed. Bump the version and mark it active, + // but there are no fields to return here. + if removed_in <= version { + return Err(MetricsError::SchemaDefinition(format!( + "Metric '{}' is removed in version \ + {}, which is not strictly after the \ + current active version, {}", + metric_name, removed_in, version, + ))); + } + CurrentVersion::Inactive { removed_in: *removed_in } + } + Some(CurrentVersion::Inactive { removed_in }) => { + return Err(MetricsError::SchemaDefinition(format!( + "Metric '{}' was already removed in \ + version {}, it cannot be removed again", + metric_name, removed_in, + ))); + } + None => { + return Err(MetricsError::SchemaDefinition(format!( + "Metric {} has no previous version, \ + it cannot be removed.", + metric_name, + ))); + } + } + } + MetricFields::Added { added_in, fields } => { + match current_version { + Some(CurrentVersion::Active { .. }) => { + return Err(MetricsError::SchemaDefinition(format!( + "Metric '{}' is already active, it \ + cannot be added again until it is removed.", + metric_name, + ))); + } + Some(CurrentVersion::Inactive { removed_in }) => { + // The metric is currently inactive, just check + // that the newly-active version is greater. + if added_in <= removed_in { + return Err(MetricsError::SchemaDefinition(format!( + "Re-added metric '{}' must appear in a later \ + version than the one in which it was removed ({})", + metric_name, removed_in, + ))); + } + CurrentVersion::Active { + version: *added_in, + fields: to_unique_field_names(metric_name, fields)?, + } + } + None => { + // There was no previous version for this + // metric, just add it. + CurrentVersion::Active { + version: *added_in, + fields: to_unique_field_names(metric_name, fields)?, + } + } + } + } + MetricFields::Versioned(new_fields) => { + match current_version { + Some(CurrentVersion::Active { version, .. }) => { + // The happy-path, we're stepping the version + // and possibly modifying the fields. + if new_fields.version <= *version { + return Err(MetricsError::SchemaDefinition(format!( + "Metric '{}' version should increment, \ + expected at least {}, found {}", + metric_name, + version.checked_add(1).expect("version < 256"), + new_fields.version, + ))); + } + CurrentVersion::Active { + version: new_fields.version, + fields: to_unique_field_names( + metric_name, + &new_fields.fields, + )?, + } + } + Some(CurrentVersion::Inactive { removed_in }) => { + // The metric has been removed in the past, it + // needs to be added again first. + return Err(MetricsError::SchemaDefinition(format!( + "Metric '{}' was removed in version {}, \ + it must be added again first", + metric_name, removed_in, + ))); + } + None => { + // The metric never existed, it must be added + // first. + return Err(MetricsError::SchemaDefinition(format!( + "Metric '{}' must be added in at its first \ + version, and can then be modified", + metric_name, + ))); + } + } + } + }; + Ok(new_version) +} + +fn to_unique_field_names( + name: &str, + fields: &[String], +) -> Result, MetricsError> { + let set: BTreeSet<_> = fields.iter().cloned().collect(); + if set.len() != fields.len() { + return Err(MetricsError::SchemaDefinition(format!( + "Object '{name}' has duplicate fields" + ))); + } + Ok(set) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extract_authz_scope_requires_relevant_field() { + assert!( + extract_authz_scope("foo", AuthzScope::Project, &BTreeSet::new()) + .is_err(), + "Project-scoped auth without a project_id field should be an error" + ); + + let schema = std::iter::once(FieldSchema { + name: "project_id".to_string(), + field_type: FieldType::Uuid, + source: FieldSource::Target, + description: String::new(), + }) + .collect(); + extract_authz_scope("foo", AuthzScope::Project, &schema).expect( + "Project-scoped auth with a project_id field should succeed", + ); + + let schema = std::iter::once(FieldSchema { + name: "project_id".to_string(), + field_type: FieldType::String, + source: FieldSource::Target, + description: String::new(), + }) + .collect(); + assert!( + extract_authz_scope("foo", AuthzScope::Project, &schema).is_err(), + "Project-scoped auth with a project_id field \ + that's not a UUID should be an error", + ); + } + + fn all_fields() -> BTreeMap { + let mut out = BTreeMap::new(); + for name in ["foo", "bar"] { + out.insert( + String::from(name), + FieldMetadata { + type_: FieldType::U8, + description: String::new(), + }, + ); + } + out + } + + #[test] + fn construct_field_schema_fails_with_reference_to_unknown_field() { + let all = all_fields(); + let target_fields = BTreeSet::new(); + let bad_name = String::from("baz"); + let metric_fields = std::iter::once(bad_name).collect(); + assert!( + construct_field_schema( + &all, + "target", + &target_fields, + "metric", + &metric_fields, + ) + .is_err(), + "Should return an error when the metric references a field \ + that doesn't exist in the global field list" + ); + } + + #[test] + fn construct_field_schema_fails_with_duplicate_field_names() { + let all = all_fields(); + let name = String::from("bar"); + let target_fields = std::iter::once(name.clone()).collect(); + let metric_fields = std::iter::once(name).collect(); + assert!(construct_field_schema( + &all, + "target", + &target_fields, + "metric", + &metric_fields, + ).is_err(), + "Should return an error when the target and metric share a field name" + ); + } + + #[test] + fn construct_field_schema_picks_up_correct_fields() { + let all = all_fields(); + let all_schema = all + .iter() + .zip([FieldSource::Metric, FieldSource::Target]) + .map(|((name, md), source)| FieldSchema { + name: name.clone(), + field_type: md.type_, + source, + description: String::new(), + }) + .collect(); + let foo = String::from("foo"); + let target_fields = std::iter::once(foo).collect(); + let bar = String::from("bar"); + let metric_fields = std::iter::once(bar).collect(); + assert_eq!( + construct_field_schema( + &all, + "target", + &target_fields, + "metric", + &metric_fields, + ) + .unwrap(), + all_schema, + "Each field is referenced exactly once, so we should return \ + the entire input set of fields" + ); + } + + #[test] + fn validate_field_name_disallows_bad_names() { + for name in ["PascalCase", "with spaces", "12345", "💖"] { + assert!( + validate_field_name( + name, + &FieldMetadata { + type_: FieldType::U8, + description: String::new() + } + ) + .is_err(), + "Field named {name} should be invalid" + ); + } + } + + #[test] + fn validate_field_name_verifies_uuid_field_names() { + assert!( + validate_field_name( + "projectid", + &FieldMetadata { + type_: FieldType::Uuid, + description: String::new() + } + ) + .is_err(), + "A Uuid field should be required to end in `_id`", + ); + for name in ["project_id", "id"] { + assert!( + validate_field_name(name, + &FieldMetadata { + type_: FieldType::Uuid, + description: String::new() + } + ).is_ok(), + "A Uuid field should be required to end in '_id' or exactly 'id'", + ); + } + } + + #[test] + fn extract_metric_fields_succeeds_with_gaps_in_versions() { + let metric_fields = MetricFields::Versioned(VersionedFields { + version: NonZeroU8::new(10).unwrap(), + fields: vec![], + }); + let current_version = Some(CurrentVersion::Active { + version: NonZeroU8::new(1).unwrap(), + fields: BTreeSet::new(), + }); + extract_metric_fields("foo", &metric_fields, ¤t_version).expect( + "Extracting metric fields should work with non-sequential \ + but increasing version numbers", + ); + } + + #[test] + fn extract_metric_fields_fails_with_non_increasing_versions() { + let metric_fields = MetricFields::Versioned(VersionedFields { + version: NonZeroU8::new(1).unwrap(), + fields: vec![], + }); + let current_version = Some(CurrentVersion::Active { + version: NonZeroU8::new(1).unwrap(), + fields: BTreeSet::new(), + }); + let res = + extract_metric_fields("foo", &metric_fields, ¤t_version); + let Err(MetricsError::SchemaDefinition(msg)) = &res else { + panic!("Expected schema definition error, found: {res:#?}"); + }; + assert!( + msg.contains("should increment"), + "Should fail when version numbers are non-increasing", + ); + } + + #[test] + fn extract_metric_fields_requires_adding_first() { + let metric_fields = MetricFields::Versioned(VersionedFields { + version: NonZeroU8::new(1).unwrap(), + fields: vec![], + }); + let current_version = None; + let res = + extract_metric_fields("foo", &metric_fields, ¤t_version); + let Err(MetricsError::SchemaDefinition(msg)) = &res else { + panic!("Expected schema definition error, found: {res:#?}"); + }; + assert!( + msg.contains("must be added in at its first version"), + "Should require that the first version of a metric explicitly \ + adds it in, before modification", + ); + + let metric_fields = MetricFields::Added { + added_in: NonZeroU8::new(1).unwrap(), + fields: vec![], + }; + let current_version = None; + extract_metric_fields("foo", &metric_fields, ¤t_version).expect( + "Should succeed when fields are added_in for their first version", + ); + } + + #[test] + fn extract_metric_fields_fails_to_add_existing_metric() { + let metric_fields = MetricFields::Added { + added_in: NonZeroU8::new(2).unwrap(), + fields: vec![], + }; + let current_version = Some(CurrentVersion::Active { + version: NonZeroU8::new(1).unwrap(), + fields: BTreeSet::new(), + }); + let res = + extract_metric_fields("foo", &metric_fields, ¤t_version); + let Err(MetricsError::SchemaDefinition(msg)) = &res else { + panic!("Expected schema definition error, found: {res:#?}"); + }; + assert!( + msg.contains("is already active"), + "Should fail when adding a metric that's already active", + ); + } + + #[test] + fn extract_metric_fields_fails_to_remove_non_existent_metric() { + let metric_fields = + MetricFields::Removed { removed_in: NonZeroU8::new(3).unwrap() }; + let current_version = Some(CurrentVersion::Inactive { + removed_in: NonZeroU8::new(1).unwrap(), + }); + let res = + extract_metric_fields("foo", &metric_fields, ¤t_version); + let Err(MetricsError::SchemaDefinition(msg)) = &res else { + panic!("Expected schema definition error, found: {res:#?}"); + }; + assert!( + msg.contains("was already removed"), + "Should fail when removing a metric that's already gone", + ); + } + + #[test] + fn load_schema_catches_metric_versions_not_added_in() { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] } + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { version = 1, fields = [] } + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let res = load_schema(contents); + let Err(MetricsError::SchemaDefinition(msg)) = &res else { + panic!("Should fail when metrics are not added in, but found: {res:#?}"); + }; + assert!( + msg.contains("must be added in at its first"), + "Error message should indicate that metrics need to be \ + added_in first, then modified" + ); + } + + #[test] + fn into_schema_list_fails_with_zero_metrics() { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] } + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [] } + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let mut def: TimeseriesDefinition = toml::from_str(contents).unwrap(); + def.metrics.clear(); + let res = def.into_schema_list(); + let Err(MetricsError::SchemaDefinition(msg)) = &res else { + panic!("Should fail with zero metrics, but found: {res:#?}"); + }; + assert!( + msg.contains("At least one metric must"), + "Error message should indicate that metrics need to be \ + added_in first, then modified" + ); + } + + #[test] + fn load_schema_fails_with_nonexistent_target_version() { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [] }, + { version = 2, fields = [] } + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let res = load_schema(contents); + let Err(MetricsError::SchemaDefinition(msg)) = &res else { + panic!( + "Should fail when a metric version refers \ + to a non-existent target version, but found: {res:#?}", + ); + }; + assert!( + msg.contains("does not have a matching version in the target"), + "Error message should indicate that the metric \ + refers to a nonexistent version in the target, found: {msg}", + ); + } + + fn assert_sequential_versions( + first: &TimeseriesSchema, + second: &TimeseriesSchema, + ) { + assert_eq!(first.timeseries_name, second.timeseries_name); + assert_eq!( + first.version.get(), + second.version.get().checked_sub(1).unwrap() + ); + assert_eq!(first.datum_type, second.datum_type); + assert_eq!(first.field_schema, second.field_schema); + } + + #[test] + fn load_schema_fills_in_late_implied_metric_versions() { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] }, + { version = 2, fields = [ "foo" ] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [] } + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let schema = load_schema(contents).unwrap(); + assert_eq!( + schema.len(), + 2, + "Should have filled in version 2 of the metric using the \ + corresponding target version", + ); + assert_sequential_versions(&schema[0], &schema[1]); + } + + #[test] + fn load_schema_fills_in_implied_metric_versions_when_last_is_modified() { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] }, + { version = 2, fields = [ "foo" ] }, + { version = 3, fields = [ "foo" ] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [] }, + { version = 3, fields = [] }, + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let schema = load_schema(contents).unwrap(); + assert_eq!( + schema.len(), + 3, + "Should have filled in version 2 of the metric using the \ + corresponding target version", + ); + assert_sequential_versions(&schema[0], &schema[1]); + assert_sequential_versions(&schema[1], &schema[2]); + } + + #[test] + fn load_schema_fills_in_implied_metric_versions() { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] }, + { version = 2, fields = [ "foo" ] }, + { version = 3, fields = [ "foo" ] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [] }, + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let schema = load_schema(contents).unwrap(); + assert_eq!( + schema.len(), + 3, + "Should have filled in versions 2 and 3 of the metric using the \ + corresponding target version", + ); + assert_sequential_versions(&schema[0], &schema[1]); + assert_sequential_versions(&schema[1], &schema[2]); + } + + #[test] + fn load_schema_fills_in_implied_metric_versions_when_last_version_is_removed( + ) { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] }, + { version = 2, fields = [ "foo" ] }, + { version = 3, fields = [ "foo" ] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [] }, + { removed_in = 3 }, + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let schema = load_schema(contents).unwrap(); + dbg!(&schema); + assert_eq!( + schema.len(), + 2, + "Should have filled in version 2 of the metric using the \ + corresponding target version", + ); + assert_sequential_versions(&schema[0], &schema[1]); + } + + #[test] + fn load_schema_skips_versions_until_metric_is_added() { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] }, + { version = 2, fields = [ "foo" ] }, + { version = 3, fields = [ "foo" ] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 3, fields = [] }, + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let schema = load_schema(contents).unwrap(); + assert_eq!( + schema.len(), + 1, + "Should have only created the last version of the timeseries" + ); + } + + #[test] + fn load_schema_fails_with_duplicate_timeseries() { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [] }, + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let res = load_schema(contents); + let Err(MetricsError::SchemaDefinition(msg)) = &res else { + panic!( + "Expected to fail with duplicated timeseries, but found {res:#?}", + ); + }; + assert!( + msg.ends_with("is duplicated"), + "Message should indicate that a timeseries name / \ + version is duplicated" + ); + } + + #[test] + fn only_support_format_version_1() { + let contents = r#" + format_version = 2 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ "foo" ] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [] }, + ] + [fields.foo] + type = "string" + description = "a field" + "#; + let res = load_schema(contents); + let Err(MetricsError::Toml(msg)) = &res else { + panic!( + "Expected to fail with bad format version, but found {res:#?}", + ); + }; + assert!( + msg.contains("Only version 1 of"), + "Message should indicate that only format version 1 \ + is supported, but found {msg:?}" + ); + } + + #[test] + fn ensures_target_has_at_least_one_field() { + let contents = r#" + format_version = 1 + + [target] + name = "target" + description = "some target" + authz_scope = "fleet" + versions = [ + { version = 1, fields = [ ] }, + ] + + [[metrics]] + name = "metric" + description = "some metric" + datum_type = "u8" + units = "count" + versions = [ + { added_in = 1, fields = [ "foo" ] }, + ] + + [fields.foo] + type = "string" + description = "a field" + "#; + let res = load_schema(contents); + let Err(MetricsError::SchemaDefinition(msg)) = &res else { + panic!( + "Expected to fail when target has zero fields, but found {res:#?}", + ); + }; + assert_eq!( + msg, "Target 'target' version 1 must have at least one field", + "Message should indicate that all targets must \ + have at least one field, but found {msg:?}", + ); + } +} diff --git a/oximeter/oximeter/src/schema.rs b/oximeter/impl/src/schema/mod.rs similarity index 82% rename from oximeter/oximeter/src/schema.rs rename to oximeter/impl/src/schema/mod.rs index 2a577fc8f1..28dbf38ab8 100644 --- a/oximeter/oximeter/src/schema.rs +++ b/oximeter/impl/src/schema/mod.rs @@ -2,10 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -// Copyright 2023 Oxide Computer Company +// Copyright 2024 Oxide Computer Company //! Tools for working with schema for fields and timeseries. +pub mod codegen; +pub mod ir; + use crate::types::DatumType; use crate::types::FieldType; use crate::types::MetricsError; @@ -21,8 +24,17 @@ use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt::Write; +use std::num::NonZeroU8; use std::path::Path; +/// Full path to the directory containing all schema. +/// +/// This is defined in this crate as the single source of truth, but not +/// re-exported outside implementation crates (e.g., not via `oximeter` or +/// `oximeter-collector`. +pub const SCHEMA_DIRECTORY: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/../oximeter/schema"); + /// The name and type information for a field of a timeseries schema. #[derive( Clone, @@ -39,6 +51,7 @@ pub struct FieldSchema { pub name: String, pub field_type: FieldType, pub source: FieldSource, + pub description: String, } /// The source from which a field is derived, the target or metric. @@ -68,7 +81,7 @@ pub enum FieldSource { Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Hash, Serialize, Deserialize, )] #[serde(try_from = "&str")] -pub struct TimeseriesName(String); +pub struct TimeseriesName(pub(crate) String); impl JsonSchema for TimeseriesName { fn schema_name() -> String { @@ -153,6 +166,24 @@ fn validate_timeseries_name(s: &str) -> Result<&str, MetricsError> { } } +/// Text descriptions for the target and metric of a timeseries. +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, Serialize)] +pub struct TimeseriesDescription { + pub target: String, + pub metric: String, +} + +/// Measurement units for timeseries samples. +#[derive(Clone, Copy, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[serde(rename_all = "snake_case")] +// TODO-completeness: Include more units, such as power / temperature. +// TODO-completeness: Decide whether and how to handle dimensional analysis +// during queries, if needed. +pub enum Units { + Count, + Bytes, +} + /// The schema for a timeseries. /// /// This includes the name of the timeseries, as well as the datum type of its metric and the @@ -160,20 +191,33 @@ fn validate_timeseries_name(s: &str) -> Result<&str, MetricsError> { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct TimeseriesSchema { pub timeseries_name: TimeseriesName, + pub description: TimeseriesDescription, pub field_schema: BTreeSet, pub datum_type: DatumType, + pub version: NonZeroU8, + pub authz_scope: AuthzScope, + pub units: Units, pub created: DateTime, } +/// Default version for timeseries schema, 1. +pub const fn default_schema_version() -> NonZeroU8 { + unsafe { NonZeroU8::new_unchecked(1) } +} + impl From<&Sample> for TimeseriesSchema { fn from(sample: &Sample) -> Self { - let timeseries_name = sample.timeseries_name.parse().unwrap(); + let timeseries_name = sample + .timeseries_name + .parse() + .expect("expected a legal timeseries name in a sample"); let mut field_schema = BTreeSet::new(); for field in sample.target_fields() { let schema = FieldSchema { name: field.name.clone(), field_type: field.value.field_type(), source: FieldSource::Target, + description: String::new(), }; field_schema.insert(schema); } @@ -182,30 +226,39 @@ impl From<&Sample> for TimeseriesSchema { name: field.name.clone(), field_type: field.value.field_type(), source: FieldSource::Metric, + description: String::new(), }; field_schema.insert(schema); } let datum_type = sample.measurement.datum_type(); - Self { timeseries_name, field_schema, datum_type, created: Utc::now() } + Self { + timeseries_name, + description: Default::default(), + field_schema, + datum_type, + version: default_schema_version(), + authz_scope: AuthzScope::Fleet, + units: Units::Count, + created: Utc::now(), + } } } impl TimeseriesSchema { /// Construct a timeseries schema from a target and metric. - pub fn new(target: &T, metric: &M) -> Self + pub fn new(target: &T, metric: &M) -> Result where T: Target, M: Metric, { - let timeseries_name = - TimeseriesName::try_from(crate::timeseries_name(target, metric)) - .unwrap(); + let timeseries_name = crate::timeseries_name(target, metric)?; let mut field_schema = BTreeSet::new(); for field in target.fields() { let schema = FieldSchema { name: field.name.clone(), field_type: field.value.field_type(), source: FieldSource::Target, + description: String::new(), }; field_schema.insert(schema); } @@ -214,11 +267,21 @@ impl TimeseriesSchema { name: field.name.clone(), field_type: field.value.field_type(), source: FieldSource::Metric, + description: String::new(), }; field_schema.insert(schema); } let datum_type = metric.datum_type(); - Self { timeseries_name, field_schema, datum_type, created: Utc::now() } + Ok(Self { + timeseries_name, + description: Default::default(), + field_schema, + datum_type, + version: default_schema_version(), + authz_scope: AuthzScope::Fleet, + units: Units::Count, + created: Utc::now(), + }) } /// Construct a timeseries schema from a sample @@ -240,11 +303,22 @@ impl TimeseriesSchema { .split_once(':') .expect("Incorrectly formatted timseries name") } + + /// Return the name of the target for this timeseries. + pub fn target_name(&self) -> &str { + self.component_names().0 + } + + /// Return the name of the metric for this timeseries. + pub fn metric_name(&self) -> &str { + self.component_names().1 + } } impl PartialEq for TimeseriesSchema { fn eq(&self, other: &TimeseriesSchema) -> bool { self.timeseries_name == other.timeseries_name + && self.version == other.version && self.datum_type == other.datum_type && self.field_schema == other.field_schema } @@ -263,6 +337,38 @@ impl PartialEq for TimeseriesSchema { const TIMESERIES_NAME_REGEX: &str = "^(([a-z]+[a-z0-9]*)(_([a-z0-9]+))*):(([a-z]+[a-z0-9]*)(_([a-z0-9]+))*)$"; +/// Authorization scope for a timeseries. +/// +/// This describes the level at which a user must be authorized to read data +/// from a timeseries. For example, fleet-scoping means the data is only visible +/// to an operator or fleet reader. Project-scoped, on the other hand, indicates +/// that a user will see data limited to the projects on which they have read +/// permissions. +#[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + Hash, + JsonSchema, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[serde(rename_all = "snake_case")] +pub enum AuthzScope { + /// Timeseries data is limited to fleet readers. + Fleet, + /// Timeseries data is limited to the authorized silo for a user. + Silo, + /// Timeseries data is limited to the authorized projects for a user. + Project, + /// The timeseries is viewable to all without limitation. + ViewableToAll, +} + /// A set of timeseries schema, useful for testing changes to targets or /// metrics. #[derive(Debug, Default, Deserialize, PartialEq, Serialize)] @@ -284,24 +390,24 @@ impl SchemaSet { &mut self, target: &T, metric: &M, - ) -> Option + ) -> Result, MetricsError> where T: Target, M: Metric, { - let new = TimeseriesSchema::new(target, metric); + let new = TimeseriesSchema::new(target, metric)?; let name = new.timeseries_name.clone(); match self.inner.entry(name) { Entry::Vacant(entry) => { entry.insert(new); - None + Ok(None) } Entry::Occupied(entry) => { let existing = entry.get(); if existing == &new { - None + Ok(None) } else { - Some(existing.clone()) + Ok(Some(existing.clone())) } } } @@ -535,7 +641,7 @@ mod tests { fn test_timeseries_schema_from_parts() { let target = MyTarget::default(); let metric = MyMetric::default(); - let schema = TimeseriesSchema::new(&target, &metric); + let schema = TimeseriesSchema::new(&target, &metric).unwrap(); assert_eq!(schema.timeseries_name, "my_target:my_metric"); let f = schema.schema_for_field("id").unwrap(); @@ -560,7 +666,7 @@ mod tests { let target = MyTarget::default(); let metric = MyMetric::default(); let sample = Sample::new(&target, &metric).unwrap(); - let schema = TimeseriesSchema::new(&target, &metric); + let schema = TimeseriesSchema::new(&target, &metric).unwrap(); let schema_from_sample = TimeseriesSchema::from(&sample); assert_eq!(schema, schema_from_sample); } @@ -586,11 +692,13 @@ mod tests { name: String::from("later"), field_type: FieldType::U64, source: FieldSource::Target, + description: String::new(), }; let metric_field = FieldSchema { name: String::from("earlier"), field_type: FieldType::U64, source: FieldSource::Metric, + description: String::new(), }; let timeseries_name: TimeseriesName = "foo:bar".parse().unwrap(); let datum_type = DatumType::U64; @@ -598,8 +706,12 @@ mod tests { [target_field.clone(), metric_field.clone()].into_iter().collect(); let expected_schema = TimeseriesSchema { timeseries_name, + description: Default::default(), field_schema, datum_type, + version: default_schema_version(), + authz_scope: AuthzScope::Fleet, + units: Units::Count, created: Utc::now(), }; @@ -627,11 +739,13 @@ mod tests { name: String::from("second"), field_type: FieldType::U64, source: FieldSource::Target, + description: String::new(), }); fields.insert(FieldSchema { name: String::from("first"), field_type: FieldType::U64, source: FieldSource::Target, + description: String::new(), }); let mut iter = fields.iter(); assert_eq!(iter.next().unwrap().name, "first"); diff --git a/oximeter/oximeter/src/test_util.rs b/oximeter/impl/src/test_util.rs similarity index 98% rename from oximeter/oximeter/src/test_util.rs rename to oximeter/impl/src/test_util.rs index 56992623d7..c2ac7b34bd 100644 --- a/oximeter/oximeter/src/test_util.rs +++ b/oximeter/impl/src/test_util.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. //! Utilities for testing the oximeter crate. -// Copyright 2021 Oxide Computer Company +// Copyright 2024 Oxide Computer Company use crate::histogram; use crate::histogram::{Histogram, Record}; diff --git a/oximeter/oximeter/src/traits.rs b/oximeter/impl/src/traits.rs similarity index 90% rename from oximeter/oximeter/src/traits.rs rename to oximeter/impl/src/traits.rs index 0934d231e3..16baa4f619 100644 --- a/oximeter/oximeter/src/traits.rs +++ b/oximeter/impl/src/traits.rs @@ -4,7 +4,7 @@ //! Traits used to describe metric data and its sources. -// Copyright 2021 Oxide Computer Company +// Copyright 2024 Oxide Computer Company use crate::histogram::Histogram; use crate::types; @@ -19,6 +19,7 @@ use chrono::DateTime; use chrono::Utc; use num::traits::One; use num::traits::Zero; +use std::num::NonZeroU8; use std::ops::Add; use std::ops::AddAssign; @@ -44,7 +45,11 @@ use std::ops::AddAssign; /// -------- /// /// ```rust -/// use oximeter::{Target, FieldType}; +/// # // Rename the impl crate so the doctests can refer to the public +/// # // `oximeter` crate, not the private impl. +/// # extern crate oximeter_impl as oximeter; +/// # use oximeter_macro_impl::*; +/// use oximeter::{traits::Target, types::FieldType}; /// use uuid::Uuid; /// /// #[derive(Target)] @@ -70,15 +75,29 @@ use std::ops::AddAssign; /// supported types. /// /// ```compile_fail +/// # // Rename the impl crate so the doctests can refer to the public +/// # // `oximeter` crate, not the private impl. +/// # extern crate oximeter_impl as oximeter; +/// # use oximeter_macro_impl::*; /// #[derive(oximeter::Target)] /// struct Bad { /// bad: f64, /// } /// ``` +/// +/// **Important:** Deriving this trait is deprecated, and will be removed in the +/// future. Instead, define your timeseries schema through the TOML format +/// described in [the crate documentation](crate), and use the code +/// generated by the `use_timeseries` macro. pub trait Target { /// Return the name of the target, which is the snake_case form of the struct's name. fn name(&self) -> &'static str; + /// Return the version of the target schema. + fn version(&self) -> NonZeroU8 { + unsafe { NonZeroU8::new_unchecked(1) } + } + /// Return the names of the target's fields, in the order in which they're defined. fn field_names(&self) -> &'static [&'static str]; @@ -141,6 +160,10 @@ pub trait Target { /// Example /// ------- /// ```rust +/// # // Rename the impl crate so the doctests can refer to the public +/// # // `oximeter` crate, not the private impl. +/// # extern crate oximeter_impl as oximeter; +/// # use oximeter_macro_impl::*; /// use chrono::Utc; /// use oximeter::Metric; /// @@ -162,6 +185,10 @@ pub trait Target { /// an unsupported type. /// /// ```compile_fail +/// # // Rename the impl crate so the doctests can refer to the public +/// # // `oximeter` crate, not the private impl. +/// # extern crate oximeter_impl as oximeter; +/// # use oximeter_macro_impl::*; /// #[derive(Metric)] /// pub struct BadType { /// field: f32, @@ -174,6 +201,11 @@ pub trait Metric { /// Return the name of the metric, which is the snake_case form of the struct's name. fn name(&self) -> &'static str; + /// Return the version of the metric schema. + fn version(&self) -> NonZeroU8 { + unsafe { NonZeroU8::new_unchecked(1) } + } + /// Return the names of the metric's fields, in the order in which they're defined. fn field_names(&self) -> &'static [&'static str]; @@ -332,6 +364,10 @@ pub use crate::histogram::HistogramSupport; /// Example /// ------- /// ```rust +/// # // Rename the impl crate so the doctests can refer to the public +/// # // `oximeter` crate, not the private impl. +/// # extern crate oximeter_impl as oximeter; +/// # use oximeter_macro_impl::*; /// use oximeter::{Datum, MetricsError, Metric, Producer, Target}; /// use oximeter::types::{Measurement, Sample, Cumulative}; /// diff --git a/oximeter/oximeter/src/types.rs b/oximeter/impl/src/types.rs similarity index 95% rename from oximeter/oximeter/src/types.rs rename to oximeter/impl/src/types.rs index 3e6ffc5442..a6518e4ad5 100644 --- a/oximeter/oximeter/src/types.rs +++ b/oximeter/impl/src/types.rs @@ -4,11 +4,12 @@ //! Types used to describe targets, metrics, and measurements. -// Copyright 2023 Oxide Computer Company +// Copyright 2024 Oxide Computer Company use crate::histogram; use crate::traits; use crate::Producer; +use crate::TimeseriesName; use bytes::Bytes; use chrono::DateTime; use chrono::Utc; @@ -24,6 +25,7 @@ use std::fmt; use std::net::IpAddr; use std::net::Ipv4Addr; use std::net::Ipv6Addr; +use std::num::NonZeroU8; use std::ops::Add; use std::ops::AddAssign; use std::sync::Arc; @@ -667,8 +669,21 @@ pub enum MetricsError { #[error("Missing datum of type {datum_type} cannot have a start time")] MissingDatumCannotHaveStartTime { datum_type: DatumType }, + #[error("Invalid timeseries name")] InvalidTimeseriesName, + + #[error("TOML deserialization error: {0}")] + Toml(String), + + #[error("Schema definition error: {0}")] + SchemaDefinition(String), + + #[error("Target version {target} does not match metric version {metric}")] + TargetMetricVersionMismatch { + target: std::num::NonZeroU8, + metric: std::num::NonZeroU8, + }, } impl From for omicron_common::api::external::Error { @@ -795,7 +810,13 @@ pub struct Sample { pub measurement: Measurement, /// The name of the timeseries this sample belongs to - pub timeseries_name: String, + pub timeseries_name: TimeseriesName, + + /// The version of the timeseries this sample belongs to + // + // TODO-cleanup: This should be removed once schema are tracked in CRDB. + #[serde(default = "::oximeter::schema::default_schema_version")] + pub timeseries_version: NonZeroU8, // Target name and fields target: FieldSet, @@ -810,7 +831,8 @@ impl PartialEq for Sample { /// Two samples are considered equal if they have equal targets and metrics, and occur at the /// same time. Importantly, the _data_ is not used during comparison. fn eq(&self, other: &Sample) -> bool { - self.target.eq(&other.target) + self.timeseries_version.eq(&other.timeseries_version) + && self.target.eq(&other.target) && self.metric.eq(&other.metric) && self.measurement.start_time().eq(&other.measurement.start_time()) && self.measurement.timestamp().eq(&other.measurement.timestamp()) @@ -831,11 +853,19 @@ impl Sample { T: traits::Target, M: traits::Metric, { + if target.version() != metric.version() { + return Err(MetricsError::TargetMetricVersionMismatch { + target: target.version(), + metric: metric.version(), + }); + } let target_fields = FieldSet::from_target(target); let metric_fields = FieldSet::from_metric(metric); Self::verify_field_names(&target_fields, &metric_fields)?; + let timeseries_name = crate::timeseries_name(target, metric)?; Ok(Self { - timeseries_name: crate::timeseries_name(target, metric), + timeseries_name, + timeseries_version: target.version(), target: target_fields, metric: metric_fields, measurement: metric.measure(timestamp), @@ -853,12 +883,20 @@ impl Sample { T: traits::Target, M: traits::Metric, { + if target.version() != metric.version() { + return Err(MetricsError::TargetMetricVersionMismatch { + target: target.version(), + metric: metric.version(), + }); + } let target_fields = FieldSet::from_target(target); let metric_fields = FieldSet::from_metric(metric); Self::verify_field_names(&target_fields, &metric_fields)?; let datum = Datum::Missing(MissingDatum::from(metric)); + let timeseries_name = crate::timeseries_name(target, metric)?; Ok(Self { - timeseries_name: crate::timeseries_name(target, metric), + timeseries_name, + timeseries_version: target.version(), target: target_fields, metric: metric_fields, measurement: Measurement { timestamp, datum }, diff --git a/oximeter/oximeter/tests/fail/failures.rs b/oximeter/impl/tests/fail/failures.rs similarity index 100% rename from oximeter/oximeter/tests/fail/failures.rs rename to oximeter/impl/tests/fail/failures.rs diff --git a/oximeter/oximeter/tests/fail/failures.stderr b/oximeter/impl/tests/fail/failures.stderr similarity index 100% rename from oximeter/oximeter/tests/fail/failures.stderr rename to oximeter/impl/tests/fail/failures.stderr diff --git a/oximeter/oximeter/tests/test_compilation.rs b/oximeter/impl/tests/test_compilation.rs similarity index 100% rename from oximeter/oximeter/tests/test_compilation.rs rename to oximeter/impl/tests/test_compilation.rs diff --git a/oximeter/instruments/Cargo.toml b/oximeter/instruments/Cargo.toml index a04e26fdaa..f51278f794 100644 --- a/oximeter/instruments/Cargo.toml +++ b/oximeter/instruments/Cargo.toml @@ -13,6 +13,8 @@ chrono = { workspace = true, optional = true } dropshot = { workspace = true, optional = true } futures = { workspace = true, optional = true } http = { workspace = true, optional = true } +kstat-rs = { workspace = true, optional = true } +libc = { workspace = true, optional = true } oximeter = { workspace = true, optional = true } slog = { workspace = true, optional = true } tokio = { workspace = true, optional = true } @@ -35,6 +37,7 @@ kstat = [ "dep:chrono", "dep:futures", "dep:kstat-rs", + "dep:libc", "dep:oximeter", "dep:slog", "dep:tokio", @@ -48,6 +51,3 @@ rand.workspace = true slog-async.workspace = true slog-term.workspace = true oximeter.workspace = true - -[target.'cfg(target_os = "illumos")'.dependencies] -kstat-rs = { workspace = true, optional = true } diff --git a/oximeter/instruments/src/kstat/link.rs b/oximeter/instruments/src/kstat/link.rs index 03397c4108..0507594056 100644 --- a/oximeter/instruments/src/kstat/link.rs +++ b/oximeter/instruments/src/kstat/link.rs @@ -15,98 +15,9 @@ use kstat_rs::Data; use kstat_rs::Kstat; use kstat_rs::Named; use oximeter::types::Cumulative; -use oximeter::Metric; use oximeter::Sample; -use oximeter::Target; -use uuid::Uuid; - -/// Information about a single physical Ethernet link on a host. -#[derive(Clone, Debug, Target)] -pub struct PhysicalDataLink { - /// The ID of the rack (cluster) containing this host. - pub rack_id: Uuid, - /// The ID of the sled itself. - pub sled_id: Uuid, - /// The serial number of the hosting sled. - pub serial: String, - /// The name of the host. - pub hostname: String, - /// The name of the link. - pub link_name: String, -} - -/// Information about a virtual Ethernet link on a host. -/// -/// Note that this is specifically for a VNIC in on the host system, not a guest -/// data link. -#[derive(Clone, Debug, Target)] -pub struct VirtualDataLink { - /// The ID of the rack (cluster) containing this host. - pub rack_id: Uuid, - /// The ID of the sled itself. - pub sled_id: Uuid, - /// The serial number of the hosting sled. - pub serial: String, - /// The name of the host, or the zone name for links in a zone. - pub hostname: String, - /// The name of the link. - pub link_name: String, -} - -/// Information about a guest virtual Ethernet link. -#[derive(Clone, Debug, Target)] -pub struct GuestDataLink { - /// The ID of the rack (cluster) containing this host. - pub rack_id: Uuid, - /// The ID of the sled itself. - pub sled_id: Uuid, - /// The serial number of the hosting sled. - pub serial: String, - /// The name of the host, or the zone name for links in a zone. - pub hostname: String, - /// The ID of the project containing the instance. - pub project_id: Uuid, - /// The ID of the instance. - pub instance_id: Uuid, - /// The name of the link. - pub link_name: String, -} - -/// The number of packets received on the link. -#[derive(Clone, Copy, Metric)] -pub struct PacketsReceived { - pub datum: Cumulative, -} - -/// The number of packets sent on the link. -#[derive(Clone, Copy, Metric)] -pub struct PacketsSent { - pub datum: Cumulative, -} - -/// The number of bytes sent on the link. -#[derive(Clone, Copy, Metric)] -pub struct BytesSent { - pub datum: Cumulative, -} - -/// The number of bytes received on the link. -#[derive(Clone, Copy, Metric)] -pub struct BytesReceived { - pub datum: Cumulative, -} -/// The number of errors received on the link. -#[derive(Clone, Copy, Metric)] -pub struct ErrorsReceived { - pub datum: Cumulative, -} - -/// The number of errors sent on the link. -#[derive(Clone, Copy, Metric)] -pub struct ErrorsSent { - pub datum: Cumulative, -} +oximeter::use_timeseries!("physical-data-link.toml"); // Helper function to extract the same kstat metrics from all link targets. fn extract_link_kstats( @@ -121,7 +32,7 @@ where let Named { name, value } = named_data; if *name == "rbytes64" { Some(value.as_u64().and_then(|x| { - let metric = BytesReceived { + let metric = physical_data_link::BytesReceived { datum: Cumulative::with_start_time(creation_time, x), }; Sample::new_with_timestamp(snapshot_time, target, &metric) @@ -129,7 +40,7 @@ where })) } else if *name == "obytes64" { Some(value.as_u64().and_then(|x| { - let metric = BytesSent { + let metric = physical_data_link::BytesSent { datum: Cumulative::with_start_time(creation_time, x), }; Sample::new_with_timestamp(snapshot_time, target, &metric) @@ -137,7 +48,7 @@ where })) } else if *name == "ipackets64" { Some(value.as_u64().and_then(|x| { - let metric = PacketsReceived { + let metric = physical_data_link::PacketsReceived { datum: Cumulative::with_start_time(creation_time, x), }; Sample::new_with_timestamp(snapshot_time, target, &metric) @@ -145,7 +56,7 @@ where })) } else if *name == "opackets64" { Some(value.as_u64().and_then(|x| { - let metric = PacketsSent { + let metric = physical_data_link::PacketsSent { datum: Cumulative::with_start_time(creation_time, x), }; Sample::new_with_timestamp(snapshot_time, target, &metric) @@ -153,7 +64,7 @@ where })) } else if *name == "ierrors" { Some(value.as_u32().and_then(|x| { - let metric = ErrorsReceived { + let metric = physical_data_link::ErrorsReceived { datum: Cumulative::with_start_time(creation_time, x.into()), }; Sample::new_with_timestamp(snapshot_time, target, &metric) @@ -161,7 +72,7 @@ where })) } else if *name == "oerrors" { Some(value.as_u32().and_then(|x| { - let metric = ErrorsSent { + let metric = physical_data_link::ErrorsSent { datum: Cumulative::with_start_time(creation_time, x.into()), }; Sample::new_with_timestamp(snapshot_time, target, &metric) @@ -177,19 +88,7 @@ trait LinkKstatTarget: KstatTarget { fn link_name(&self) -> &str; } -impl LinkKstatTarget for PhysicalDataLink { - fn link_name(&self) -> &str { - &self.link_name - } -} - -impl LinkKstatTarget for VirtualDataLink { - fn link_name(&self) -> &str { - &self.link_name - } -} - -impl LinkKstatTarget for GuestDataLink { +impl LinkKstatTarget for physical_data_link::PhysicalDataLink { fn link_name(&self) -> &str { &self.link_name } @@ -225,7 +124,7 @@ where } } -#[cfg(test)] +#[cfg(all(test, target_os = "illumos"))] mod tests { use super::*; use crate::kstat::sampler::KstatPath; @@ -325,17 +224,17 @@ mod tests { fn test_physical_datalink() { let link = TestEtherstub::new(); let sn = String::from("BRM000001"); - let dl = PhysicalDataLink { + let dl = physical_data_link::PhysicalDataLink { rack_id: RACK_ID, sled_id: SLED_ID, - serial: sn.clone(), - hostname: sn, - link_name: link.name.to_string(), + serial: sn.clone().into(), + hostname: sn.into(), + link_name: link.name.clone().into(), }; let ctl = Ctl::new().unwrap(); let ctl = ctl.update().unwrap(); let mut kstat = ctl - .filter(Some("link"), Some(0), Some(dl.link_name.as_str())) + .filter(Some("link"), Some(0), Some(link.name.as_str())) .next() .unwrap(); let creation_time = hrtime_to_utc(kstat.ks_crtime).unwrap(); @@ -349,12 +248,12 @@ mod tests { let mut sampler = KstatSampler::new(&test_logger()).unwrap(); let sn = String::from("BRM000001"); let link = TestEtherstub::new(); - let dl = PhysicalDataLink { + let dl = physical_data_link::PhysicalDataLink { rack_id: RACK_ID, sled_id: SLED_ID, - serial: sn.clone(), - hostname: sn, - link_name: link.name.to_string(), + serial: sn.clone().into(), + hostname: sn.into(), + link_name: link.name.clone().into(), }; let details = CollectionDetails::never(Duration::from_secs(1)); let id = sampler.add_target(dl, details).await.unwrap(); @@ -397,12 +296,12 @@ mod tests { KstatSampler::with_sample_limit(&test_logger(), limit).unwrap(); let sn = String::from("BRM000001"); let link = TestEtherstub::new(); - let dl = PhysicalDataLink { + let dl = physical_data_link::PhysicalDataLink { rack_id: RACK_ID, sled_id: SLED_ID, - serial: sn.clone(), - hostname: sn, - link_name: link.name.to_string(), + serial: sn.clone().into(), + hostname: sn.into(), + link_name: link.name.to_string().into(), }; let details = CollectionDetails::never(Duration::from_secs(1)); sampler.add_target(dl, details).await.unwrap(); @@ -464,12 +363,12 @@ mod tests { let sn = String::from("BRM000001"); let link = TestEtherstub::new(); info!(log, "created test etherstub"; "name" => &link.name); - let dl = PhysicalDataLink { + let dl = physical_data_link::PhysicalDataLink { rack_id: RACK_ID, sled_id: SLED_ID, - serial: sn.clone(), - hostname: sn, - link_name: link.name.to_string(), + serial: sn.clone().into(), + hostname: sn.into(), + link_name: link.name.to_string().into(), }; let collection_interval = Duration::from_secs(1); let expiry = Duration::from_secs(1); @@ -521,12 +420,12 @@ mod tests { let sn = String::from("BRM000001"); let link = TestEtherstub::new(); info!(log, "created test etherstub"; "name" => &link.name); - let dl = PhysicalDataLink { + let dl = physical_data_link::PhysicalDataLink { rack_id: RACK_ID, sled_id: SLED_ID, - serial: sn.clone(), - hostname: sn, - link_name: link.name.to_string(), + serial: sn.clone().into(), + hostname: sn.into(), + link_name: link.name.to_string().into(), }; let collection_interval = Duration::from_secs(1); let expiry = Duration::from_secs(1); @@ -570,12 +469,12 @@ mod tests { name: link.name.clone(), }; info!(log, "created test etherstub"; "name" => &link.name); - let dl = PhysicalDataLink { + let dl = physical_data_link::PhysicalDataLink { rack_id: RACK_ID, sled_id: SLED_ID, - serial: sn.clone(), - hostname: sn, - link_name: link.name.to_string(), + serial: sn.clone().into(), + hostname: sn.into(), + link_name: link.name.to_string().into(), }; let collection_interval = Duration::from_secs(1); let expiry = Duration::from_secs(1); @@ -617,12 +516,12 @@ mod tests { name: link.name.clone(), }; info!(log, "created test etherstub"; "name" => &link.name); - let dl = PhysicalDataLink { + let dl = physical_data_link::PhysicalDataLink { rack_id: RACK_ID, sled_id: SLED_ID, - serial: sn.clone(), - hostname: sn, - link_name: link.name.to_string(), + serial: sn.clone().into(), + hostname: sn.into(), + link_name: link.name.to_string().into(), }; let collection_interval = Duration::from_secs(1); let expiry = Duration::from_secs(1); diff --git a/oximeter/instruments/src/kstat/mod.rs b/oximeter/instruments/src/kstat/mod.rs index c792a51408..a5020b9b61 100644 --- a/oximeter/instruments/src/kstat/mod.rs +++ b/oximeter/instruments/src/kstat/mod.rs @@ -97,6 +97,22 @@ pub use sampler::KstatSampler; pub use sampler::TargetId; pub use sampler::TargetStatus; +cfg_if::cfg_if! { + if #[cfg(all(test, target_os = "illumos"))] { + type Timestamp = tokio::time::Instant; + #[inline(always)] + fn now() -> Timestamp { + tokio::time::Instant::now() + } + } else { + type Timestamp = chrono::DateTime; + #[inline(always)] + fn now() -> Timestamp { + chrono::Utc::now() + } + } +} + /// The reason a kstat target was expired and removed from a sampler. #[derive(Clone, Copy, Debug)] pub enum ExpirationReason { @@ -114,10 +130,7 @@ pub struct Expiration { /// The last error before expiration. pub error: Box, /// The time at which the expiration occurred. - #[cfg(test)] - pub expired_at: tokio::time::Instant, - #[cfg(not(test))] - pub expired_at: DateTime, + pub expired_at: Timestamp, } /// Errors resulting from reporting kernel statistics. @@ -191,7 +204,7 @@ pub trait KstatTarget: /// Convert from a high-res timestamp into UTC, if possible. pub fn hrtime_to_utc(hrtime: i64) -> Result, Error> { let utc_now = Utc::now(); - let hrtime_now = unsafe { gethrtime() }; + let hrtime_now = get_hires_time(); match hrtime_now.cmp(&hrtime) { Ordering::Equal => Ok(utc_now), Ordering::Less => { @@ -262,7 +275,27 @@ impl<'a> ConvertNamedData for NamedData<'a> { } } -#[link(name = "c")] -extern "C" { - fn gethrtime() -> i64; +/// Return a high-resolution monotonic timestamp, in nanoseconds since an +/// arbitrary point in the past. +/// +/// This is equivalent to `gethrtime(3C)` on illumos, and `clock_gettime()` with +/// an equivalent clock source on other platforms. +pub fn get_hires_time() -> i64 { + // NOTE: See `man clock_gettime`, but this is an alias for `CLOCK_HIGHRES`, + // and is the same source that underlies `gethrtime()`, which this API is + // intended to emulate on other platforms. + #[cfg(target_os = "illumos")] + const SOURCE: libc::clockid_t = libc::CLOCK_MONOTONIC; + #[cfg(not(target_os = "illumos"))] + const SOURCE: libc::clockid_t = libc::CLOCK_MONOTONIC_RAW; + let mut tp = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + if unsafe { libc::clock_gettime(SOURCE, &mut tp as *mut _) } == 0 { + const NANOS_PER_SEC: i64 = 1_000_000_000; + tp.tv_sec + .checked_mul(NANOS_PER_SEC) + .and_then(|nsec| nsec.checked_add(tp.tv_nsec)) + .unwrap_or(0) + } else { + 0 + } } diff --git a/oximeter/instruments/src/kstat/sampler.rs b/oximeter/instruments/src/kstat/sampler.rs index af1b3ba7cf..74770a6225 100644 --- a/oximeter/instruments/src/kstat/sampler.rs +++ b/oximeter/instruments/src/kstat/sampler.rs @@ -4,6 +4,8 @@ //! Generate oximeter samples from kernel statistics. +use super::now; +use super::Timestamp; use crate::kstat::hrtime_to_utc; use crate::kstat::Error; use crate::kstat::Expiration; @@ -41,9 +43,6 @@ use tokio::time::interval; use tokio::time::sleep; use tokio::time::Sleep; -#[cfg(test)] -use tokio::time::Instant; - // The `KstatSampler` generates some statistics about its own operation, mostly // for surfacing failures to collect and dropped samples. mod self_stats { @@ -165,12 +164,7 @@ pub enum TargetStatus { /// The target is currently being collected from normally. /// /// The timestamp of the last collection is included. - Ok { - #[cfg(test)] - last_collection: Option, - #[cfg(not(test))] - last_collection: Option>, - }, + Ok { last_collection: Option }, /// The target has been expired. /// /// The details about the expiration are included. @@ -178,10 +172,7 @@ pub enum TargetStatus { reason: ExpirationReason, // NOTE: The error is a string, because it's not cloneable. error: String, - #[cfg(test)] - expired_at: Instant, - #[cfg(not(test))] - expired_at: DateTime, + expired_at: Timestamp, }, } @@ -204,7 +195,7 @@ enum Request { /// Remove a target. RemoveTarget { id: TargetId, reply_tx: oneshot::Sender> }, /// Return the creation times of all tracked / extant kstats. - #[cfg(test)] + #[cfg(all(test, target_os = "illumos"))] CreationTimes { reply_tx: oneshot::Sender>>, }, @@ -218,15 +209,9 @@ struct SampledKstat { /// The details around collection and expiration behavior. details: CollectionDetails, /// The time at which we _added_ this target to the sampler. - #[cfg(test)] - time_added: Instant, - #[cfg(not(test))] - time_added: DateTime, + time_added: Timestamp, /// The last time we successfully collected from the target. - #[cfg(test)] - time_of_last_collection: Option, - #[cfg(not(test))] - time_of_last_collection: Option>, + time_of_last_collection: Option, /// Attempts since we last successfully collected from the target. attempts_since_last_collection: usize, } @@ -384,7 +369,7 @@ fn hostname() -> Option { } /// Stores the number of samples taken, used for testing. -#[cfg(test)] +#[cfg(all(test, target_os = "illumos"))] pub(crate) struct SampleCounts { pub total: usize, pub overflow: usize, @@ -420,7 +405,8 @@ impl KstatSamplerWorker { /// kstats at their intervals. Samples will be pushed onto the queue. async fn run( mut self, - #[cfg(test)] sample_count_tx: mpsc::UnboundedSender, + #[cfg(all(test, target_os = "illumos"))] + sample_count_tx: mpsc::UnboundedSender, ) { let mut sample_timeouts = FuturesUnordered::new(); let mut creation_prune_interval = @@ -487,7 +473,7 @@ impl KstatSamplerWorker { // Send the total number of samples we've actually // taken and the number we've appended over to any // testing code which might be listening. - #[cfg(test)] + #[cfg(all(test, target_os = "illumos"))] sample_count_tx.send(SampleCounts { total: n_samples, overflow: n_overflow_samples, @@ -635,7 +621,7 @@ impl KstatSamplerWorker { ), } } - #[cfg(test)] + #[cfg(all(test, target_os = "illumos"))] Request::CreationTimes { reply_tx } => { debug!(self.log, "request for creation times"); reply_tx.send(self.creation_times.clone()).unwrap(); @@ -734,13 +720,7 @@ impl KstatSamplerWorker { .collect::, _>>(); match kstats { Ok(k) if !k.is_empty() => { - cfg_if::cfg_if! { - if #[cfg(test)] { - sampled_kstat.time_of_last_collection = Some(Instant::now()); - } else { - sampled_kstat.time_of_last_collection = Some(Utc::now()); - } - } + sampled_kstat.time_of_last_collection = Some(now()); sampled_kstat.attempts_since_last_collection = 0; sampled_kstat.target.to_samples(&k).map(Option::Some) } @@ -769,17 +749,10 @@ impl KstatSamplerWorker { if sampled_kstat.attempts_since_last_collection >= n_attempts { - cfg_if::cfg_if! { - if #[cfg(test)] { - let expired_at = Instant::now(); - } else { - let expired_at = Utc::now(); - } - } return Err(Error::Expired(Expiration { reason: ExpirationReason::Attempts(n_attempts), error: Box::new(e), - expired_at, + expired_at: now(), })); } } @@ -790,18 +763,12 @@ impl KstatSamplerWorker { .time_of_last_collection .unwrap_or(sampled_kstat.time_added); let expire_at = start + duration; - cfg_if::cfg_if! { - if #[cfg(test)] { - let now = Instant::now(); - } else { - let now = Utc::now(); - } - } - if now >= expire_at { + let now_ = now(); + if now_ >= expire_at { return Err(Error::Expired(Expiration { reason: ExpirationReason::Duration(duration), error: Box::new(e), - expired_at: now, + expired_at: now_, })); } } @@ -1019,18 +986,10 @@ impl KstatSamplerWorker { None => {} } self.ensure_creation_times_for_target(&*target)?; - - cfg_if::cfg_if! { - if #[cfg(test)] { - let time_added = Instant::now(); - } else { - let time_added = Utc::now(); - } - } let item = SampledKstat { target, details, - time_added, + time_added: now(), time_of_last_collection: None, attempts_since_last_collection: 0, }; @@ -1076,7 +1035,7 @@ pub struct KstatSampler { outbox: mpsc::Sender, self_stat_rx: Arc>>, _worker_task: Arc>, - #[cfg(test)] + #[cfg(all(test, target_os = "illumos"))] sample_count_rx: Arc>>, } @@ -1110,7 +1069,7 @@ impl KstatSampler { samples.clone(), limit, )?; - #[cfg(test)] + #[cfg(all(test, target_os = "illumos"))] let (sample_count_rx, _worker_task) = { let (sample_count_tx, sample_count_rx) = mpsc::unbounded_channel(); ( @@ -1118,14 +1077,14 @@ impl KstatSampler { Arc::new(tokio::task::spawn(worker.run(sample_count_tx))), ) }; - #[cfg(not(test))] + #[cfg(not(all(test, target_os = "illumos")))] let _worker_task = Arc::new(tokio::task::spawn(worker.run())); Ok(Self { samples, outbox, self_stat_rx: Arc::new(Mutex::new(self_stat_rx)), _worker_task, - #[cfg(test)] + #[cfg(all(test, target_os = "illumos"))] sample_count_rx, }) } @@ -1174,7 +1133,7 @@ impl KstatSampler { } /// Return the number of samples pushed by the sampling task, if any. - #[cfg(test)] + #[cfg(all(test, target_os = "illumos"))] pub(crate) fn sample_counts(&self) -> Option { match self.sample_count_rx.lock().unwrap().try_recv() { Ok(c) => Some(c), @@ -1184,7 +1143,7 @@ impl KstatSampler { } /// Return the creation times for all tracked kstats. - #[cfg(test)] + #[cfg(all(test, target_os = "illumos"))] pub(crate) async fn creation_times( &self, ) -> BTreeMap> { diff --git a/oximeter/instruments/src/lib.rs b/oximeter/instruments/src/lib.rs index c1f839c85d..521034e423 100644 --- a/oximeter/instruments/src/lib.rs +++ b/oximeter/instruments/src/lib.rs @@ -9,5 +9,5 @@ #[cfg(feature = "http-instruments")] pub mod http; -#[cfg(all(feature = "kstat", target_os = "illumos"))] +#[cfg(feature = "kstat")] pub mod kstat; diff --git a/oximeter/oximeter-macro-impl/src/lib.rs b/oximeter/oximeter-macro-impl/src/lib.rs index f110d00e69..499cd82d0a 100644 --- a/oximeter/oximeter-macro-impl/src/lib.rs +++ b/oximeter/oximeter-macro-impl/src/lib.rs @@ -153,6 +153,10 @@ fn build_shared_methods(item_name: &Ident, fields: &[&Field]) -> TokenStream { #name } + fn version(&self) -> ::std::num::NonZeroU8 { + unsafe { ::std::num::NonZeroU8::new_unchecked(1) } + } + fn field_names(&self) -> &'static [&'static str] { &[#(#names),*] } diff --git a/oximeter/oximeter/Cargo.toml b/oximeter/oximeter/Cargo.toml index 0fe52bc4ac..c04d1bd3ae 100644 --- a/oximeter/oximeter/Cargo.toml +++ b/oximeter/oximeter/Cargo.toml @@ -9,25 +9,14 @@ license = "MPL-2.0" workspace = true [dependencies] -bytes = { workspace = true, features = [ "serde" ] } +anyhow.workspace = true +clap.workspace = true chrono.workspace = true -float-ord.workspace = true -num.workspace = true -omicron-common.workspace = true +omicron-workspace-hack.workspace = true +oximeter-impl.workspace = true oximeter-macro-impl.workspace = true -regex.workspace = true -schemars = { workspace = true, features = [ "uuid1", "bytes", "chrono" ] } -serde.workspace = true -serde_json.workspace = true -strum.workspace = true -thiserror.workspace = true +oximeter-timeseries-macro.workspace = true +prettyplease.workspace = true +syn.workspace = true +toml.workspace = true uuid.workspace = true -omicron-workspace-hack.workspace = true - -[dev-dependencies] -approx.workspace = true -rand = { workspace = true, features = ["std_rng"] } -rand_distr.workspace = true -rstest.workspace = true -serde_json.workspace = true -trybuild.workspace = true diff --git a/oximeter/oximeter/schema/physical-data-link.toml b/oximeter/oximeter/schema/physical-data-link.toml new file mode 100644 index 0000000000..d526aa6af1 --- /dev/null +++ b/oximeter/oximeter/schema/physical-data-link.toml @@ -0,0 +1,95 @@ +format_version = 1 + +[target] +name = "physical_data_link" +description = "A physical network link on a compute sled" +authz_scope = "fleet" +versions = [ + { version = 1, fields = [ "rack_id", "sled_id", "hostname", "serial", "link_name" ] }, + # This is the intended next version, but actual schema updates are not yet + # supported. This is left here as an example and breadcrumb to implement + # that update in the future. + #{ version = 2, fields = [ "rack_id", "sled_id", "serial", "model", "revision", "link_name" ] }, +] + +[fields.rack_id] +type = "uuid" +description = "UUID for the link's sled" + +[fields.sled_id] +type = "uuid" +description = "UUID for the link's sled" + +[fields.hostname] +type = "string" +description = "Hostname of the link's sled" + +[fields.model] +type = "string" +description = "Model number of the link's sled" + +[fields.revision] +type = "u32" +description = "Revision number of the sled" + +[fields.serial] +type = "string" +description = "Serial number of the sled" + +[fields.link_name] +type = "string" +description = "Name of the physical data link" + +[[metrics]] +name = "bytes_sent" +description = "Number of bytes sent on the link" +units = "bytes" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "bytes_received" +description = "Number of bytes received on the link" +units = "bytes" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "packets_sent" +description = "Number of packets sent on the link" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "packets_received" +description = "Number of packets received on the link" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "errors_sent" +description = "Number of errors encountered when sending on the link" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "errors_received" +description = "Number of errors encountered when receiving on the link" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] diff --git a/oximeter/oximeter/src/bin/oximeter-schema.rs b/oximeter/oximeter/src/bin/oximeter-schema.rs new file mode 100644 index 0000000000..14fb31b1e8 --- /dev/null +++ b/oximeter/oximeter/src/bin/oximeter-schema.rs @@ -0,0 +1,97 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2024 Oxide Computer Company + +//! CLI tool to understand timeseries schema + +use anyhow::Context as _; +use clap::Parser; +use clap::Subcommand; +use oximeter::schema::ir::TimeseriesDefinition; +use std::num::NonZeroU8; +use std::path::PathBuf; + +#[derive(Debug, Parser)] +struct Args { + #[command(subcommand)] + cmd: Cmd, + /// The path to the schema definition TOML file. + path: PathBuf, +} + +#[derive(Debug, Subcommand)] +enum Cmd { + /// Print the intermediate representation parsed from the schema file + Ir, + + /// Print the derived timeseries schema. + Schema { + /// Show the schema for a specified timeseries by name. + /// + /// If not provided, all timeseries are printed. + #[arg(short, long)] + timeseries: Option, + + /// Show the schema for a specified version. + /// + /// If not provided, all versions are shown. + #[arg(short, long)] + version: Option, + }, + + /// Print the Rust code that would be emitted in the macro format. + Emit, +} + +fn main() -> anyhow::Result<()> { + let args = Args::try_parse()?; + let contents = std::fs::read_to_string(&args.path).with_context(|| { + format!("failed to read from {}", args.path.display()) + })?; + match args.cmd { + Cmd::Ir => { + let def: TimeseriesDefinition = toml::from_str(&contents)?; + println!("{def:#?}"); + } + Cmd::Schema { timeseries, version } => { + let schema = oximeter_impl::schema::ir::load_schema(&contents)?; + match (timeseries, version) { + (None, None) => { + for each in schema.into_iter() { + println!("{each:#?}"); + } + } + (None, Some(version)) => { + for each in + schema.into_iter().filter(|s| s.version == version) + { + println!("{each:#?}"); + } + } + (Some(name), None) => { + for each in + schema.into_iter().filter(|s| s.timeseries_name == name) + { + println!("{each:#?}"); + } + } + (Some(name), Some(version)) => { + for each in schema.into_iter().filter(|s| { + s.timeseries_name == name && s.version == version + }) { + println!("{each:#?}"); + } + } + } + } + Cmd::Emit => { + let code = oximeter::schema::codegen::use_timeseries(&contents)?; + let formatted = + prettyplease::unparse(&syn::parse_file(&format!("{code}"))?); + println!("{formatted}"); + } + } + Ok(()) +} diff --git a/oximeter/oximeter/src/lib.rs b/oximeter/oximeter/src/lib.rs index cd5c5adf8c..9dd8fab47a 100644 --- a/oximeter/oximeter/src/lib.rs +++ b/oximeter/oximeter/src/lib.rs @@ -2,6 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +// Copyright 2024 Oxide Computer Company + //! Tools for generating and collecting metric data in the Oxide rack. //! //! Overview @@ -18,33 +20,121 @@ //! sensor. A _target_ is the thing being measured -- the service or the DIMM, in these cases. Both //! targets and metrics may have key-value pairs associated with them, called _fields_. //! -//! Motivating example -//! ------------------ +//! Defining timeseries schema +//! -------------------------- //! -//! ```rust -//! use uuid::Uuid; -//! use oximeter::{types::Cumulative, Metric, Target}; +//! Creating a timeseries starts by defining its schema. This includes the name +//! for the target and metric, as well as other metadata such as descriptions, +//! data types, and version numbers. Let's start by looking at an example: +//! +//! ```text +//! [target] +//! name = "foo" +//! description = "Statistics about the foo server" +//! authz_scope = "fleet" +//! versions = [ +//! { version = 1, fields = ["name", "id"], +//! ] +//! +//! [[metrics]] +//! name = "total_requests" +//! description = "The cumulative number of requests served" +//! datum_type = "cumulative_u64" +//! units = "count" +//! versions = [ +//! { added_in = 1, fields = ["route", "method", "response_code"], +//! ] +//! +//! [fields.name] +//! type = "string" +//! description = "The name of this server" +//! +//! [fields.id] +//! type = "uuid" +//! description = "Unique ID of this server" +//! +//! [fields.route] +//! type = "string" +//! description = "The route used in the HTTP request" +//! +//! [fields.method] +//! type = "string" +//! description = "The method used in the HTTP request" +//! +//! [fields.response_code] +//! type = u16 +//! description = "Status code the server responded with" +//! ``` +//! +//! In this case, our target is an HTTP server, which we identify with the +//! fields "name" and "id". Those fields are described in the `fields` TOML key, +//! and referred to by name in the target definition. A target also needs to +//! have a description and an _authorization scope_, which describes the +//! visibility of the timeseries and its data. See +//! [`crate::schema::AuthzScope`] for details. +//! +//! A target can have one or more metrics defined for it. As with the target, a +//! metric also has a name and description, and additionally a datum type and +//! units. It may also have fields, again referred to by name. +//! +//! This file should live in the `oximeter/schema` subdirectory, so that it can +//! be used to generate Rust code for producing data. +//! +//! Versions +//! -------- +//! +//! Both targets and metrics have version numbers associated with them. For a +//! target, these numbers must be the numbers 1, 2, 3, ... (increasing, no +//! gaps). As the target or any of its metrics evolves, these numbers are +//! incremented, and the new fields for that version are specified. +//! +//! The fields on the metrics may be specified a bit more flexibly. The first +//! version they appear in must have the form: `{ added_in = X, fields = [ ... +//! ] }`. After, the versions may be omitted, meaning that the previous version +//! of the metric is unchanged; or the metric may be removed entirely with a +//! version like `{ removed_in = Y }`. +//! +//! In all cases, the TOML definition is checked for consistency. Any existing +//! metric versions must match up with a target version, and they must have +//! distinct field names (targets and metrics cannot share fields). +//! +//! Using these primitives, fields may be added, removed, or renamed, with one +//! caveat: a field may not **change type**. +//! +//! Generated code +//! -------------- //! -//! #[derive(Target)] +//! This TOML definition can be used in a number of ways, but the most relevant +//! is to actually produce data from the resulting timeseries. This can be done +//! with the `[use_timeseries]` proc-macro, like this: +//! +//! ```ignore +//! oximeter::use_timeseries!("http-server.toml"); +//! ``` +//! +//! The macro will first validate the timeseries definition, and then generate +//! Rust code like the following: +//! +//! ```rust +//! #[derive(oximeter::Target)] //! struct HttpServer { //! name: String, -//! id: Uuid, +//! id: uuid::Uuid, //! } //! -//! #[derive(Metric)] +//! #[derive(oximeter::Metric)] //! struct TotalRequests { //! route: String, //! method: String, -//! response_code: i64, +//! response_code: u16, //! #[datum] -//! total: Cumulative, +//! total: oximeter::types::Cumulative, //! } //! ``` //! -//! In this case, our target is some HTTP server, which we identify with the fields "name" and -//! "id". The metric of interest is the total count of requests, by route/method/response code, -//! over time. The [`types::Cumulative`] type keeps track of cumulative scalar values, an integer -//! in this case. +//! This code can be used to create **samples** from this timeseries. The target +//! and metric structs can be filled in with the timeseries's _fields_, and the +//! _datum_ may be populated to generate new samples. //! //! Datum, measurement, and samples //! ------------------------------- @@ -95,47 +185,93 @@ //! `Producer`s may be registered with the same `ProducerServer`, each with potentially different //! sampling intervals. -// Copyright 2023 Oxide Computer Company - -pub use oximeter_macro_impl::*; - -// Export the current crate as `oximeter`. The macros defined in `oximeter-macro-impl` generate -// code referring to symbols like `oximeter::traits::Target`. In consumers of this crate, that's -// fine, but internally there _is_ no crate named `oximeter`, it's just `self` or `crate`. -// -// See https://github.com/rust-lang/rust/pull/55275 for the PR introducing this fix, which links to -// lots of related issues and discussion. -extern crate self as oximeter; +pub use oximeter_impl::*; +pub use oximeter_timeseries_macro::use_timeseries; -pub mod histogram; -pub mod quantile; -pub mod schema; -pub mod test_util; -pub mod traits; -pub mod types; +#[cfg(test)] +mod test { + use oximeter_impl::schema::ir::load_schema; + use oximeter_impl::schema::{FieldSource, SCHEMA_DIRECTORY}; + use oximeter_impl::TimeseriesSchema; + use std::collections::BTreeMap; + use std::fs; -pub use quantile::Quantile; -pub use quantile::QuantileError; -pub use schema::FieldSchema; -pub use schema::TimeseriesName; -pub use schema::TimeseriesSchema; -pub use traits::Metric; -pub use traits::Producer; -pub use traits::Target; -pub use types::Datum; -pub use types::DatumType; -pub use types::Field; -pub use types::FieldType; -pub use types::FieldValue; -pub use types::Measurement; -pub use types::MetricsError; -pub use types::Sample; + /// This test checks that changes to timeseries schema are all consistent. + /// + /// Timeseries schema are described in a TOML format that makes it relatively + /// easy to add new versions of the timeseries. Those definitions are ingested + /// at compile-time and checked for self-consistency, but it's still possible + /// for two unrelated definitions to conflict. This test catches those. + #[test] + fn timeseries_schema_consistency() { + let mut all_schema = BTreeMap::new(); + for entry in fs::read_dir(SCHEMA_DIRECTORY).unwrap() { + let entry = entry.unwrap(); + println!( + "examining timeseries schema from: '{}'", + entry.path().canonicalize().unwrap().display() + ); + let contents = fs::read_to_string(entry.path()).unwrap(); + let list = load_schema(&contents).unwrap_or_else(|_| { + panic!( + "Expected a valid timeseries definition in {}", + entry.path().canonicalize().unwrap().display() + ) + }); + println!("found {} schema", list.len()); + for schema in list.into_iter() { + let key = (schema.timeseries_name.clone(), schema.version); + if let Some(dup) = all_schema.insert(key, schema.clone()) { + panic!( + "Timeseries '{}' version {} is duplicated.\ + \noriginal:\n{}\nduplicate:{}\n", + schema.timeseries_name, + schema.version, + pretty_print_schema(&schema), + pretty_print_schema(&dup), + ); + } + } + } + } -/// Construct the timeseries name for a Target and Metric. -pub fn timeseries_name(target: &T, metric: &M) -> String -where - T: Target, - M: Metric, -{ - format!("{}:{}", target.name(), metric.name()) + fn pretty_print_schema(schema: &TimeseriesSchema) -> String { + use std::fmt::Write; + let mut out = String::new(); + writeln!(out, " name: {}", schema.timeseries_name).unwrap(); + writeln!(out, " version: {}", schema.version).unwrap(); + writeln!(out, " target").unwrap(); + writeln!(out, " description: {}", schema.description.target).unwrap(); + writeln!(out, " fields:").unwrap(); + for field in schema + .field_schema + .iter() + .filter(|field| field.source == FieldSource::Target) + { + writeln!( + out, + " {} ({}): {}", + field.name, field.field_type, field.description + ) + .unwrap(); + } + writeln!(out, " metric").unwrap(); + writeln!(out, " description: {}", schema.description.metric).unwrap(); + writeln!(out, " fields:").unwrap(); + for field in schema + .field_schema + .iter() + .filter(|field| field.source == FieldSource::Metric) + { + writeln!( + out, + " {} ({}): {}", + field.name, field.field_type, field.description + ) + .unwrap(); + } + writeln!(out, " datum type: {}", schema.datum_type).unwrap(); + writeln!(out, " units: {:?}", schema.units).unwrap(); + out + } } diff --git a/oximeter/timeseries-macro/Cargo.toml b/oximeter/timeseries-macro/Cargo.toml new file mode 100644 index 0000000000..db591aed06 --- /dev/null +++ b/oximeter/timeseries-macro/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "oximeter-timeseries-macro" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] +omicron-workspace-hack.workspace = true +oximeter-impl.workspace = true +proc-macro2.workspace = true +quote.workspace = true +syn.workspace = true + +[lints] +workspace = true diff --git a/oximeter/timeseries-macro/src/lib.rs b/oximeter/timeseries-macro/src/lib.rs new file mode 100644 index 0000000000..317a8533a4 --- /dev/null +++ b/oximeter/timeseries-macro/src/lib.rs @@ -0,0 +1,106 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2024 Oxide Computer Company + +//! Procedural macro to emi Rust code matching a TOML timeseries definition. + +extern crate proc_macro; + +use oximeter_impl::schema::SCHEMA_DIRECTORY; + +/// Generate code to use the timeseries from one target. +/// +/// This macro accepts a single filename, which should be a file in the +/// `oximeter/schema` subdirectory, containing a valid timeseries definition. It +/// attempts parse the file and generate code used to produce samples from those +/// timeseries. It will generate a submodule named by the target in the file, +/// with a Rust struct for the target and each metric defined in the file. +#[proc_macro] +pub fn use_timeseries( + tokens: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + match syn::parse::(tokens) { + Ok(token) => { + let path = match extract_timeseries_path(&token) { + Ok(path) => path, + Err(err) => return err, + }; + + // Now that we have a verified file name only, look up the contents + // within the schema directory; validate it; and generate code from + // it. + let contents = match std::fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + let msg = format!( + "Failed to read timeseries schema \ + from file '{}': {:?}", + path.display(), + e, + ); + return syn::Error::new(token.span(), msg) + .into_compile_error() + .into(); + } + }; + match oximeter_impl::schema::codegen::use_timeseries(&contents) { + Ok(toks) => { + let path_ = path.display().to_string(); + return quote::quote! { + /// Include the schema file itself to ensure we recompile + /// when that changes. + const _: &str = include_str!(#path_); + #toks + } + .into(); + } + Err(e) => { + let msg = format!( + "Failed to generate timeseries types \ + from '{}': {:?}", + path.display(), + e, + ); + return syn::Error::new(token.span(), msg) + .into_compile_error() + .into(); + } + } + } + Err(e) => return e.into_compile_error().into(), + } +} + +// Extract the full path to the timeseries definition, from the macro input +// tokens. We currently only allow a filename with no other path components, to +// avoid looking in directories other than the `SCHEMA_DIRECTORY`. +fn extract_timeseries_path( + token: &syn::LitStr, +) -> Result { + let make_err = || { + let path = std::path::Path::new(SCHEMA_DIRECTORY) + .canonicalize() + .map(|p| p.display().to_string()) + .unwrap_or_else(|_| SCHEMA_DIRECTORY.to_string()); + let msg = format!( + "Input must be a valid filename with no \ + path components and directly within the \ + schema directory '{}'", + path, + ); + Err(syn::Error::new(token.span(), msg).into_compile_error().into()) + }; + let value = token.value(); + if value.is_empty() { + return make_err(); + } + let Some(filename) = std::path::Path::new(&value).file_name() else { + return make_err(); + }; + if filename != value.as_str() { + return make_err(); + } + Ok(std::path::Path::new(SCHEMA_DIRECTORY).join(&filename)) +} diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 04ba5d8d31..a42a22efd8 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -195,6 +195,7 @@ }, "checker": { "description": "Checker to apply to incoming messages.", + "default": null, "type": [ "string", "null" @@ -209,6 +210,7 @@ }, "shaper": { "description": "Shaper to apply to outgoing messages.", + "default": null, "type": [ "string", "null" @@ -319,6 +321,7 @@ }, "local_pref": { "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": [ "integer", "null" @@ -328,6 +331,7 @@ }, "md5_auth_key": { "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": [ "string", "null" @@ -335,6 +339,7 @@ }, "min_ttl": { "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": [ "integer", "null" @@ -344,6 +349,7 @@ }, "multi_exit_discriminator": { "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": [ "integer", "null" @@ -357,6 +363,7 @@ }, "remote_asn": { "description": "Require that a peer has a specified ASN.", + "default": null, "type": [ "integer", "null" @@ -366,6 +373,7 @@ }, "vlan_id": { "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": [ "integer", "null" @@ -894,6 +902,7 @@ }, "vlan_id": { "description": "The VLAN id associated with this route.", + "default": null, "type": [ "integer", "null" @@ -997,6 +1006,7 @@ }, "vlan_id": { "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": [ "integer", "null" diff --git a/sled-agent/src/metrics.rs b/sled-agent/src/metrics.rs index 62eaaf6154..fcb260e93a 100644 --- a/sled-agent/src/metrics.rs +++ b/sled-agent/src/metrics.rs @@ -8,30 +8,23 @@ use omicron_common::api::internal::nexus::ProducerEndpoint; use omicron_common::api::internal::nexus::ProducerKind; use oximeter::types::MetricsError; use oximeter::types::ProducerRegistry; +use oximeter_instruments::kstat::link; +use oximeter_instruments::kstat::CollectionDetails; +use oximeter_instruments::kstat::Error as KstatError; +use oximeter_instruments::kstat::KstatSampler; +use oximeter_instruments::kstat::TargetId; use oximeter_producer::LogConfig; use oximeter_producer::Server as ProducerServer; use sled_hardware_types::Baseboard; use slog::Logger; +use std::collections::BTreeMap; use std::net::Ipv6Addr; use std::net::SocketAddr; use std::sync::Arc; +use std::sync::Mutex; use std::time::Duration; use uuid::Uuid; -cfg_if::cfg_if! { - if #[cfg(target_os = "illumos")] { - use oximeter_instruments::kstat::link; - use oximeter_instruments::kstat::CollectionDetails; - use oximeter_instruments::kstat::Error as KstatError; - use oximeter_instruments::kstat::KstatSampler; - use oximeter_instruments::kstat::TargetId; - use std::collections::BTreeMap; - use std::sync::Mutex; - } else { - use anyhow::anyhow; - } -} - /// The interval on which we ask `oximeter` to poll us for metric data. pub(crate) const METRIC_COLLECTION_INTERVAL: Duration = Duration::from_secs(30); @@ -44,14 +37,9 @@ const METRIC_REQUEST_MAX_SIZE: usize = 10 * 1024 * 1024; /// An error during sled-agent metric production. #[derive(Debug, thiserror::Error)] pub enum Error { - #[cfg(target_os = "illumos")] #[error("Kstat-based metric failure")] Kstat(#[source] KstatError), - #[cfg(not(target_os = "illumos"))] - #[error("Kstat-based metric failure")] - Kstat(#[source] anyhow::Error), - #[error("Failed to insert metric producer into registry")] Registry(#[source] MetricsError), @@ -70,7 +58,6 @@ pub enum Error { // Basic metadata about the sled agent used when publishing metrics. #[derive(Clone, Debug)] -#[cfg_attr(not(target_os = "illumos"), allow(dead_code))] struct SledIdentifiers { sled_id: Uuid, rack_id: Uuid, @@ -86,13 +73,9 @@ struct SledIdentifiers { // pattern, but until we have more statistics, it's not clear whether that's // worth it right now. #[derive(Clone)] -// NOTE: The ID fields aren't used on non-illumos systems, rather than changing -// the name of fields that are not yet used. -#[cfg_attr(not(target_os = "illumos"), allow(dead_code))] pub struct MetricsManager { metadata: Arc, _log: Logger, - #[cfg(target_os = "illumos")] kstat_sampler: KstatSampler, // TODO-scalability: We may want to generalize this to store any kind of // tracked target, and use a naming scheme that allows us pick out which @@ -103,7 +86,6 @@ pub struct MetricsManager { // for disks or memory. If we wanted to guarantee uniqueness, we could // namespace them internally, e.g., `"datalink:{link_name}"` would be the // real key. - #[cfg(target_os = "illumos")] tracked_links: Arc>>, producer_server: Arc, } @@ -122,23 +104,16 @@ impl MetricsManager { ) -> Result { let producer_server = start_producer_server(&log, sled_id, sled_address)?; - - cfg_if::cfg_if! { - if #[cfg(target_os = "illumos")] { - let kstat_sampler = KstatSampler::new(&log).map_err(Error::Kstat)?; - producer_server - .registry() - .register_producer(kstat_sampler.clone()) - .map_err(Error::Registry)?; - let tracked_links = Arc::new(Mutex::new(BTreeMap::new())); - } - } + let kstat_sampler = KstatSampler::new(&log).map_err(Error::Kstat)?; + producer_server + .registry() + .register_producer(kstat_sampler.clone()) + .map_err(Error::Registry)?; + let tracked_links = Arc::new(Mutex::new(BTreeMap::new())); Ok(Self { metadata: Arc::new(SledIdentifiers { sled_id, rack_id, baseboard }), _log: log, - #[cfg(target_os = "illumos")] kstat_sampler, - #[cfg(target_os = "illumos")] tracked_links, producer_server, }) @@ -178,7 +153,6 @@ fn start_producer_server( ProducerServer::start(&config).map(Arc::new).map_err(Error::ProducerServer) } -#[cfg(target_os = "illumos")] impl MetricsManager { /// Track metrics for a physical datalink. pub async fn track_physical_link( @@ -187,12 +161,12 @@ impl MetricsManager { interval: Duration, ) -> Result<(), Error> { let hostname = hostname()?; - let link = link::PhysicalDataLink { + let link = link::physical_data_link::PhysicalDataLink { rack_id: self.metadata.rack_id, sled_id: self.metadata.sled_id, - serial: self.serial_number(), - hostname, - link_name: link_name.as_ref().to_string(), + serial: self.serial_number().into(), + hostname: hostname.into(), + link_name: link_name.as_ref().to_string().into(), }; let details = CollectionDetails::never(interval); let id = self @@ -224,29 +198,6 @@ impl MetricsManager { } } - /// Track metrics for a virtual datalink. - #[allow(dead_code)] - pub async fn track_virtual_link( - &self, - link_name: impl AsRef, - hostname: impl AsRef, - interval: Duration, - ) -> Result<(), Error> { - let link = link::VirtualDataLink { - rack_id: self.metadata.rack_id, - sled_id: self.metadata.sled_id, - serial: self.serial_number(), - hostname: hostname.as_ref().to_string(), - link_name: link_name.as_ref().to_string(), - }; - let details = CollectionDetails::never(interval); - self.kstat_sampler - .add_target(link, details) - .await - .map(|_| ()) - .map_err(Error::Kstat) - } - // Return the serial number out of the baseboard, if one exists. fn serial_number(&self) -> String { match &self.metadata.baseboard { @@ -257,48 +208,7 @@ impl MetricsManager { } } -#[cfg(not(target_os = "illumos"))] -impl MetricsManager { - /// Track metrics for a physical datalink. - pub async fn track_physical_link( - &self, - _link_name: impl AsRef, - _interval: Duration, - ) -> Result<(), Error> { - Err(Error::Kstat(anyhow!( - "kstat metrics are not supported on this platform" - ))) - } - - /// Stop tracking metrics for a datalink. - /// - /// This works for both physical and virtual links. - #[allow(dead_code)] - pub async fn stop_tracking_link( - &self, - _link_name: impl AsRef, - ) -> Result<(), Error> { - Err(Error::Kstat(anyhow!( - "kstat metrics are not supported on this platform" - ))) - } - - /// Track metrics for a virtual datalink. - #[allow(dead_code)] - pub async fn track_virtual_link( - &self, - _link_name: impl AsRef, - _hostname: impl AsRef, - _interval: Duration, - ) -> Result<(), Error> { - Err(Error::Kstat(anyhow!( - "kstat metrics are not supported on this platform" - ))) - } -} - // Return the current hostname if possible. -#[cfg(target_os = "illumos")] fn hostname() -> Result { // See netdb.h const MAX_LEN: usize = 256; diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 1b21b72495..0dca1a904e 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -83,13 +83,13 @@ pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] petgraph = { version = "0.6.5", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } predicates = { version = "3.1.0" } -proc-macro2 = { version = "1.0.82" } +proc-macro2 = { version = "1.0.86" } regex = { version = "1.10.4" } regex-automata = { version = "0.4.6", default-features = false, features = ["dfa", "hybrid", "meta", "nfa", "perf", "unicode"] } regex-syntax = { version = "0.8.3" } reqwest = { version = "0.11.27", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] } ring = { version = "0.17.8", features = ["std"] } -schemars = { version = "0.8.20", features = ["bytes", "chrono", "uuid1"] } +schemars = { version = "0.8.21", features = ["bytes", "chrono", "uuid1"] } scopeguard = { version = "1.2.0" } semver = { version = "1.0.23", features = ["serde"] } serde = { version = "1.0.203", features = ["alloc", "derive", "rc"] } @@ -101,7 +101,7 @@ smallvec = { version = "1.13.2", default-features = false, features = ["const_ne spin = { version = "0.9.8" } string_cache = { version = "0.8.7" } subtle = { version = "2.5.0" } -syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.64", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.68", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } time = { version = "0.3.36", features = ["formatting", "local-offset", "macros", "parsing"] } tokio = { version = "1.37.0", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } @@ -187,13 +187,13 @@ pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] petgraph = { version = "0.6.5", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } predicates = { version = "3.1.0" } -proc-macro2 = { version = "1.0.82" } +proc-macro2 = { version = "1.0.86" } regex = { version = "1.10.4" } regex-automata = { version = "0.4.6", default-features = false, features = ["dfa", "hybrid", "meta", "nfa", "perf", "unicode"] } regex-syntax = { version = "0.8.3" } reqwest = { version = "0.11.27", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] } ring = { version = "0.17.8", features = ["std"] } -schemars = { version = "0.8.20", features = ["bytes", "chrono", "uuid1"] } +schemars = { version = "0.8.21", features = ["bytes", "chrono", "uuid1"] } scopeguard = { version = "1.2.0" } semver = { version = "1.0.23", features = ["serde"] } serde = { version = "1.0.203", features = ["alloc", "derive", "rc"] } @@ -206,7 +206,7 @@ spin = { version = "0.9.8" } string_cache = { version = "0.8.7" } subtle = { version = "2.5.0" } syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] } -syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.64", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.68", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } time = { version = "0.3.36", features = ["formatting", "local-offset", "macros", "parsing"] } time-macros = { version = "0.2.18", default-features = false, features = ["formatting", "parsing"] } tokio = { version = "1.37.0", features = ["full", "test-util"] }