From a8a49c396b90083f248b103580724dd42dca297d Mon Sep 17 00:00:00 2001 From: bnaecker Date: Tue, 14 Nov 2023 14:25:31 -0800 Subject: [PATCH 01/11] Stop collecting Propolis metrics on instance stop (#4495) When Nexus responds to a sled-agent notification that the instance is stopped and its Propolis server is gone, hard-delete the assignment record and ask `oximeter` to stop collecting from it. --- nexus/db-model/src/oximeter_info.rs | 5 +- nexus/db-queries/src/db/datastore/oximeter.rs | 41 +++++++++++++-- nexus/src/app/instance.rs | 19 ++++++- nexus/src/app/oximeter.rs | 52 +++++++++++++++++++ 4 files changed, 110 insertions(+), 7 deletions(-) diff --git a/nexus/db-model/src/oximeter_info.rs b/nexus/db-model/src/oximeter_info.rs index ac30384c59..39bde98ea8 100644 --- a/nexus/db-model/src/oximeter_info.rs +++ b/nexus/db-model/src/oximeter_info.rs @@ -8,7 +8,7 @@ use chrono::{DateTime, Utc}; use nexus_types::internal_api; use uuid::Uuid; -/// Message used to notify Nexus that this oximeter instance is up and running. +/// A record representing a registered `oximeter` collector. #[derive(Queryable, Insertable, Debug, Clone, Copy)] #[diesel(table_name = oximeter)] pub struct OximeterInfo { @@ -18,8 +18,9 @@ pub struct OximeterInfo { pub time_created: DateTime, /// When this resource was last modified. pub time_modified: DateTime, - /// The address on which this oximeter instance listens for requests + /// The address on which this `oximeter` instance listens for requests. pub ip: ipnetwork::IpNetwork, + /// The port on which this `oximeter` instance listens for requests. pub port: SqlU16, } diff --git a/nexus/db-queries/src/db/datastore/oximeter.rs b/nexus/db-queries/src/db/datastore/oximeter.rs index c9b3a59b05..55b650ea53 100644 --- a/nexus/db-queries/src/db/datastore/oximeter.rs +++ b/nexus/db-queries/src/db/datastore/oximeter.rs @@ -21,7 +21,20 @@ use omicron_common::api::external::ResourceType; use uuid::Uuid; impl DataStore { - // Create a record for a new Oximeter instance + /// Lookup an oximeter instance by its ID. + pub async fn oximeter_lookup( + &self, + id: &Uuid, + ) -> Result { + use db::schema::oximeter::dsl; + dsl::oximeter + .find(*id) + .first_async(&*self.pool_connection_unauthorized().await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + /// Create a record for a new Oximeter instance pub async fn oximeter_create( &self, info: &OximeterInfo, @@ -55,7 +68,7 @@ impl DataStore { Ok(()) } - // List the oximeter collector instances + /// List the oximeter collector instances pub async fn oximeter_list( &self, page_params: &DataPageParams<'_, Uuid>, @@ -69,7 +82,7 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } - // Create a record for a new producer endpoint + /// Create a record for a new producer endpoint pub async fn producer_endpoint_create( &self, producer: &ProducerEndpoint, @@ -102,7 +115,27 @@ impl DataStore { Ok(()) } - // List the producer endpoint records by the oximeter instance to which they're assigned. + /// Delete a record for a producer endpoint, by its ID. + /// + /// This is idempotent, and deleting a record that is already removed is a + /// no-op. If the record existed, then the ID of the `oximeter` collector is + /// returned. If there was no record, `None` is returned. + pub async fn producer_endpoint_delete( + &self, + id: &Uuid, + ) -> Result, Error> { + use db::schema::metric_producer::dsl; + diesel::delete(dsl::metric_producer.find(*id)) + .returning(dsl::oximeter_id) + .get_result_async::( + &*self.pool_connection_unauthorized().await?, + ) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + /// List the producer endpoint records by the oximeter instance to which they're assigned. pub async fn producers_list_by_oximeter_id( &self, oximeter_id: Uuid, diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 17d033c5a0..923bb1777e 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1320,7 +1320,9 @@ impl super::Nexus { .await?; // If the supplied instance state indicates that the instance no longer - // has an active VMM, attempt to delete the virtual provisioning record + // has an active VMM, attempt to delete the virtual provisioning record, + // and the assignment of the Propolis metric producer to an oximeter + // collector. // // As with updating networking state, this must be done before // committing the new runtime state to the database: once the DB is @@ -1338,6 +1340,21 @@ impl super::Nexus { (&new_runtime_state.instance_state.gen).into(), ) .await?; + + // TODO-correctness: The `notify_instance_updated` method can run + // concurrently with itself in some situations, such as where a + // sled-agent attempts to update Nexus about a stopped instance; + // that times out; and it makes another request to a different + // Nexus. The call to `unassign_producer` is racy in those + // situations, and we may end with instances with no metrics. + // + // This unfortunate case should be handled as part of + // instance-lifecycle improvements, notably using a reliable + // persistent workflow to correctly update the oximete assignment as + // an instance's state changes. + // + // Tracked in https://github.com/oxidecomputer/omicron/issues/3742. + self.unassign_producer(instance_id).await?; } // Write the new instance and VMM states back to CRDB. This needs to be diff --git a/nexus/src/app/oximeter.rs b/nexus/src/app/oximeter.rs index bc947cf4bc..03f833b087 100644 --- a/nexus/src/app/oximeter.rs +++ b/nexus/src/app/oximeter.rs @@ -187,6 +187,58 @@ impl super::Nexus { Ok(()) } + /// Idempotently un-assign a producer from an oximeter collector. + pub(crate) async fn unassign_producer( + &self, + id: &Uuid, + ) -> Result<(), Error> { + if let Some(collector_id) = + self.db_datastore.producer_endpoint_delete(id).await? + { + debug!( + self.log, + "deleted metric producer assignment"; + "producer_id" => %id, + "collector_id" => %collector_id, + ); + let oximeter_info = + self.db_datastore.oximeter_lookup(&collector_id).await?; + let address = + SocketAddr::new(oximeter_info.ip.ip(), *oximeter_info.port); + let client = self.build_oximeter_client(&id, address); + if let Err(e) = client.producer_delete(&id).await { + error!( + self.log, + "failed to delete producer from collector"; + "producer_id" => %id, + "collector_id" => %collector_id, + "address" => %address, + "error" => ?e, + ); + return Err(Error::internal_error( + format!("failed to delete producer from collector: {e:?}") + .as_str(), + )); + } else { + debug!( + self.log, + "successfully deleted producer from collector"; + "producer_id" => %id, + "collector_id" => %collector_id, + "address" => %address, + ); + Ok(()) + } + } else { + trace!( + self.log, + "un-assigned non-existent metric producer"; + "producer_id" => %id, + ); + Ok(()) + } + } + /// Returns a results from the timeseries DB based on the provided query /// parameters. /// From f513182346be09008f6ccc0114386db8e3f90153 Mon Sep 17 00:00:00 2001 From: Patrick Mooney Date: Tue, 14 Nov 2023 17:40:21 -0600 Subject: [PATCH 02/11] Update propolis mock-server and client deps With the propolis mock-server split out from the "real" propolis-server, and the handmade types cleaned out of the propolis-client library, we can now free ourselves of the somewhat circular dependency situation. --- Cargo.lock | 1022 +++-------------- Cargo.toml | 9 +- sled-agent/Cargo.toml | 5 +- sled-agent/src/common/disk.rs | 2 +- sled-agent/src/common/instance.rs | 8 +- sled-agent/src/instance.rs | 23 +- sled-agent/src/params.rs | 4 +- sled-agent/src/sim/disk.rs | 2 +- sled-agent/src/sim/http_entrypoints_pantry.rs | 2 +- sled-agent/src/sim/instance.rs | 11 +- sled-agent/src/sim/sled_agent.rs | 42 +- sled-agent/src/sim/storage.rs | 2 +- tools/update_crucible.sh | 1 - workspace-hack/Cargo.toml | 29 +- 14 files changed, 221 insertions(+), 941 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f103fdd2c..6b3007b86e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,21 +52,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "aes-gcm-siv" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "polyval", - "subtle", - "zeroize", -] - [[package]] name = "ahash" version = "0.8.3" @@ -87,12 +72,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" - [[package]] name = "android-tzdata" version = "0.1.1" @@ -190,16 +169,6 @@ dependencies = [ "syn 2.0.32", ] -[[package]] -name = "api_identity" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - [[package]] name = "approx" version = "0.5.1" @@ -248,12 +217,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "ascii" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" - [[package]] name = "ascii-canvas" version = "3.0.0" @@ -296,17 +259,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-recursion" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - [[package]] name = "async-stream" version = "0.3.5" @@ -495,7 +447,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" dependencies = [ "bhyve_api_sys", "libc", @@ -505,21 +457,12 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" dependencies = [ "libc", "strum", ] -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bindgen" version = "0.65.1" @@ -558,12 +501,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" -[[package]] -name = "bit_field" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" - [[package]] name = "bitfield" version = "0.14.0" @@ -585,26 +522,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bitstruct" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b10c3912af09af44ea1dafe307edb5ed374b2a32658eb610e372270c9017b4" -dependencies = [ - "bitstruct_derive", -] - -[[package]] -name = "bitstruct_derive" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35fd19022c2b750d14eb9724c204d08ab7544570105b3b466d8a9f2f3feded27" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "bitvec" version = "1.0.1" @@ -678,7 +595,7 @@ dependencies = [ "derive_more", "hex", "hkdf", - "omicron-common 0.1.0", + "omicron-common", "omicron-rpaths", "omicron-test-utils", "omicron-workspace-hack", @@ -705,7 +622,7 @@ name = "bootstrap-agent-client" version = "0.1.0" dependencies = [ "ipnetwork", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "progenitor", "regress", @@ -907,7 +824,6 @@ version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ - "jobserver", "libc", ] @@ -1112,26 +1028,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" -[[package]] -name = "const_format" -version = "0.2.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c990efc7a285731f9a4378d81aff2f0e85a2c8781a05ef0f8baa8dac54d0ff48" -dependencies = [ - "const_format_proc_macros", -] - -[[package]] -name = "const_format_proc_macros" -version = "0.2.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e026b6ce194a874cb9cf32cd5772d1ef9767cc8fcb5765948d74f37a9d8b2bf6" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - [[package]] name = "constant_time_eq" version = "0.2.6" @@ -1191,18 +1087,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpuid_profile_config" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "propolis", - "serde", - "serde_derive", - "thiserror", - "toml 0.7.8", -] - [[package]] name = "crc" version = "3.0.1" @@ -1383,51 +1267,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crucible" -version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "aes-gcm-siv", - "anyhow", - "async-recursion", - "async-trait", - "base64 0.21.5", - "bytes", - "chrono", - "crucible-client-types", - "crucible-common", - "crucible-protocol", - "crucible-workspace-hack", - "dropshot", - "futures", - "futures-core", - "itertools 0.11.0", - "libc", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter-producer 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "rand 0.8.5", - "rand_chacha 0.3.1", - "reqwest", - "ringbuffer", - "schemars", - "serde", - "serde_json", - "slog", - "slog-async", - "slog-dtrace", - "slog-term", - "tokio", - "tokio-rustls", - "tokio-util", - "toml 0.8.8", - "tracing", - "usdt", - "uuid", - "version_check", -] - [[package]] name = "crucible-agent-client" version = "0.0.1" @@ -1444,47 +1283,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "crucible-client-types" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "base64 0.21.5", - "crucible-workspace-hack", - "schemars", - "serde", - "serde_json", - "uuid", -] - -[[package]] -name = "crucible-common" -version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "anyhow", - "atty", - "crucible-workspace-hack", - "nix 0.26.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rusqlite", - "rustls-pemfile", - "schemars", - "serde", - "serde_json", - "slog", - "slog-async", - "slog-bunyan", - "slog-dtrace", - "slog-term", - "tempfile", - "thiserror", - "tokio-rustls", - "toml 0.8.8", - "twox-hash", - "uuid", - "vergen", -] - [[package]] name = "crucible-pantry-client" version = "0.0.1" @@ -1502,23 +1300,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "crucible-protocol" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "anyhow", - "bincode", - "bytes", - "crucible-common", - "crucible-workspace-hack", - "num_enum 0.7.0", - "schemars", - "serde", - "tokio-util", - "uuid", -] - [[package]] name = "crucible-smf" version = "0.0.0" @@ -1735,7 +1516,7 @@ version = "0.1.0" dependencies = [ "anyhow", "either", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "omicron-zone-package", "progenitor", @@ -1970,15 +1751,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f77af9e75578c1ab34f5f04545a8b05be0c36fbd7a9bb3cf2d2a971e435fdbb9" -[[package]] -name = "dladm" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "libc", - "strum", -] - [[package]] name = "dlpi" version = "0.2.0" @@ -1986,7 +1758,7 @@ source = "git+https://github.com/oxidecomputer/dlpi-sys#1d587ea98cf2d36f1b1624b0 dependencies = [ "libc", "libdlpi-sys", - "num_enum 0.5.11", + "num_enum", "pretty-hex 0.2.1", "thiserror", "tokio", @@ -2000,7 +1772,7 @@ dependencies = [ "camino", "chrono", "clap 4.4.3", - "dns-service-client 0.1.0", + "dns-service-client", "dropshot", "expectorate", "http", @@ -2044,22 +1816,6 @@ dependencies = [ "slog", ] -[[package]] -name = "dns-service-client" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "chrono", - "http", - "progenitor", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "uuid", -] - [[package]] name = "doc-comment" version = "0.3.3" @@ -2304,19 +2060,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "env_logger" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - [[package]] name = "env_logger" version = "0.10.0" @@ -2334,15 +2077,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "erased-serde" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "837c0466252947ada828b975e12daf82e18bb5444e4df87be6038d4469e2a3d2" -dependencies = [ - "serde", -] - [[package]] name = "errno" version = "0.3.2" @@ -2381,12 +2115,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fastrand" version = "2.0.0" @@ -2691,7 +2419,7 @@ dependencies = [ "futures", "gateway-client", "gateway-messages", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "reqwest", "serde", @@ -2753,7 +2481,7 @@ dependencies = [ "hubpack 0.1.2", "hubtools", "lru-cache", - "nix 0.26.2 (git+https://github.com/jgallagher/nix?branch=r0.26-illumos)", + "nix", "once_cell", "paste", "serde", @@ -2846,19 +2574,6 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" -[[package]] -name = "git2" -version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044" -dependencies = [ - "bitflags 1.3.2", - "libc", - "libgit2-sys", - "log", - "url", -] - [[package]] name = "glob" version = "0.3.1" @@ -2943,19 +2658,6 @@ name = "hashbrown" version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" -dependencies = [ - "ahash", - "allocator-api2", -] - -[[package]] -name = "hashlink" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" -dependencies = [ - "hashbrown 0.14.2", -] [[package]] name = "headers" @@ -3339,7 +3041,7 @@ source = "git+https://github.com/oxidecomputer/illumos-devinfo?branch=main#4323b dependencies = [ "anyhow", "libc", - "num_enum 0.5.11", + "num_enum", ] [[package]] @@ -3363,7 +3065,7 @@ dependencies = [ "libc", "macaddr", "mockall", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "opte-ioctl", "oxide-vpc", @@ -3472,7 +3174,7 @@ dependencies = [ "ipcc-key-value", "itertools 0.11.0", "libc", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "partial-io", @@ -3524,7 +3226,7 @@ dependencies = [ "expectorate", "hyper", "installinator-common", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "openapi-lint", @@ -3570,12 +3272,12 @@ dependencies = [ "assert_matches", "chrono", "dns-server", - "dns-service-client 0.1.0", + "dns-service-client", "dropshot", "expectorate", "futures", "hyper", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "progenitor", @@ -3591,25 +3293,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "internal-dns" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "anyhow", - "chrono", - "dns-service-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "futures", - "hyper", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "reqwest", - "slog", - "thiserror", - "trust-dns-proto", - "trust-dns-resolver", - "uuid", -] - [[package]] name = "internal-dns-cli" version = "0.1.0" @@ -3617,8 +3300,8 @@ dependencies = [ "anyhow", "clap 4.4.3", "dropshot", - "internal-dns 0.1.0", - "omicron-common 0.1.0", + "internal-dns", + "omicron-common", "omicron-workspace-hack", "slog", "tokio", @@ -3642,7 +3325,7 @@ version = "0.1.0" dependencies = [ "ciborium", "libc", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "proptest", "serde", @@ -3720,15 +3403,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" -[[package]] -name = "jobserver" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.64" @@ -3753,7 +3427,7 @@ version = "0.1.0" dependencies = [ "async-trait", "hkdf", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "secrecy", "sha3", @@ -3853,18 +3527,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b024e211b1b371da58cd69e4fb8fa4ed16915edcc0e2e1fb04ac4bad61959f25" -[[package]] -name = "libgit2-sys" -version = "0.15.2+1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a80df2e11fb4a61f4ba2ab42dbe7f74468da143f1a75c74e11dee7c813f694fa" -dependencies = [ - "cc", - "libc", - "libz-sys", - "pkg-config", -] - [[package]] name = "libloading" version = "0.7.4" @@ -3891,7 +3553,7 @@ dependencies = [ "colored", "dlpi", "libc", - "num_enum 0.5.11", + "num_enum", "nvpair", "nvpair-sys", "rusty-doors", @@ -3900,16 +3562,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "libsqlite3-sys" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" -dependencies = [ - "pkg-config", - "vcpkg", -] - [[package]] name = "libsw" version = "3.3.0" @@ -3941,18 +3593,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "libz-sys" -version = "1.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linear-map" version = "1.2.0" @@ -4013,7 +3653,7 @@ dependencies = [ "const-oid", "crc-any", "der", - "env_logger 0.10.0", + "env_logger", "hex", "log", "lpc55_areas", @@ -4122,7 +3762,7 @@ version = "0.1.0" dependencies = [ "anyhow", "either", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "omicron-zone-package", "progenitor", @@ -4285,8 +3925,8 @@ dependencies = [ "chrono", "futures", "ipnetwork", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", + "omicron-common", + "omicron-passwords", "omicron-workspace-hack", "progenitor", "regress", @@ -4300,26 +3940,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "nexus-client" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "chrono", - "futures", - "ipnetwork", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "omicron-passwords 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "progenitor", - "regress", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "uuid", -] - [[package]] name = "nexus-db-model" version = "0.1.0" @@ -4336,11 +3956,11 @@ dependencies = [ "nexus-defaults", "nexus-types", "omicron-certificates", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", + "omicron-common", + "omicron-passwords", "omicron-rpaths", "omicron-workspace-hack", - "parse-display 0.8.2", + "parse-display", "pq-sys", "rand 0.8.5", "ref-cast", @@ -4379,7 +3999,7 @@ dependencies = [ "http", "hyper", "hyper-rustls", - "internal-dns 0.1.0", + "internal-dns", "ipnetwork", "itertools 0.11.0", "lazy_static", @@ -4389,8 +4009,8 @@ dependencies = [ "nexus-inventory", "nexus-test-utils", "nexus-types", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", + "omicron-common", + "omicron-passwords", "omicron-rpaths", "omicron-sled-agent", "omicron-test-utils", @@ -4398,7 +4018,7 @@ dependencies = [ "openapiv3 1.0.3", "openssl", "oso", - "oximeter 0.1.0", + "oximeter", "paste", "pem 1.1.1", "petgraph", @@ -4432,7 +4052,7 @@ version = "0.1.0" dependencies = [ "ipnetwork", "lazy_static", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "rand 0.8.5", "serde_json", @@ -4463,7 +4083,7 @@ version = "0.1.0" dependencies = [ "async-trait", "nexus-types", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "slog", "uuid", @@ -4480,25 +4100,25 @@ dependencies = [ "chrono", "crucible-agent-client", "dns-server", - "dns-service-client 0.1.0", + "dns-service-client", "dropshot", "gateway-messages", "gateway-test-utils", "headers", "http", "hyper", - "internal-dns 0.1.0", + "internal-dns", "nexus-db-queries", "nexus-test-interface", "nexus-types", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", + "omicron-common", + "omicron-passwords", "omicron-sled-agent", "omicron-test-utils", "omicron-workspace-hack", - "oximeter 0.1.0", + "oximeter", "oximeter-collector", - "oximeter-producer 0.1.0", + "oximeter-producer", "serde", "serde_json", "serde_urlencoded", @@ -4521,17 +4141,17 @@ name = "nexus-types" version = "0.1.0" dependencies = [ "anyhow", - "api_identity 0.1.0", + "api_identity", "base64 0.21.5", "chrono", - "dns-service-client 0.1.0", + "dns-service-client", "futures", "gateway-client", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", + "omicron-common", + "omicron-passwords", "omicron-workspace-hack", "openssl", - "parse-display 0.8.2", + "parse-display", "schemars", "serde", "serde_json", @@ -4549,20 +4169,6 @@ dependencies = [ "smallvec 1.11.0", ] -[[package]] -name = "nix" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" -dependencies = [ - "bitflags 1.3.2", - "cfg-if 1.0.0", - "libc", - "memoffset 0.7.1", - "pin-utils", - "static_assertions", -] - [[package]] name = "nix" version = "0.26.2" @@ -4728,16 +4334,7 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" dependencies = [ - "num_enum_derive 0.5.11", -] - -[[package]] -name = "num_enum" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70bf6736f74634d299d00086f02986875b3c2d924781a6a2cb6c201e73da0ceb" -dependencies = [ - "num_enum_derive 0.7.0", + "num_enum_derive", ] [[package]] @@ -4752,18 +4349,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "num_enum_derive" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ea360eafe1022f7cc56cd7b869ed57330fb2453d0c7831d99b74c65d2f5597" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.32", -] - [[package]] name = "num_threads" version = "0.1.6" @@ -4832,7 +4417,7 @@ version = "0.1.0" dependencies = [ "display-error-chain", "foreign-types 0.3.2", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "openssl", @@ -4846,7 +4431,7 @@ name = "omicron-common" version = "0.1.0" dependencies = [ "anyhow", - "api_identity 0.1.0", + "api_identity", "async-trait", "backoff", "camino", @@ -4859,69 +4444,29 @@ dependencies = [ "http", "ipnetwork", "lazy_static", - "libc", - "macaddr", - "omicron-workspace-hack", - "parse-display 0.8.2", - "progenitor", - "proptest", - "rand 0.8.5", - "regress", - "reqwest", - "schemars", - "semver 1.0.20", - "serde", - "serde_human_bytes", - "serde_json", - "serde_urlencoded", - "serde_with", - "slog", - "strum", - "test-strategy", - "thiserror", - "tokio", - "tokio-postgres", - "toml 0.8.8", - "uuid", -] - -[[package]] -name = "omicron-common" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "anyhow", - "api_identity 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "async-trait", - "backoff", - "camino", - "chrono", - "dropshot", - "futures", - "hex", - "http", - "hyper", - "ipnetwork", - "lazy_static", + "libc", "macaddr", - "parse-display 0.7.0", + "omicron-workspace-hack", + "parse-display", "progenitor", + "proptest", "rand 0.8.5", + "regress", "reqwest", - "ring 0.16.20", "schemars", "semver 1.0.20", "serde", - "serde_derive", "serde_human_bytes", "serde_json", + "serde_urlencoded", "serde_with", "slog", "strum", + "test-strategy", "thiserror", "tokio", "tokio-postgres", - "toml 0.7.8", + "toml 0.8.8", "uuid", ] @@ -4955,7 +4500,7 @@ dependencies = [ "libc", "nexus-test-interface", "nexus-test-utils", - "omicron-common 0.1.0", + "omicron-common", "omicron-nexus", "omicron-rpaths", "omicron-test-utils", @@ -4990,7 +4535,7 @@ dependencies = [ "hyper", "illumos-utils", "ipcc-key-value", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "once_cell", @@ -5031,7 +4576,7 @@ dependencies = [ "crucible-pantry-client", "diesel", "dns-server", - "dns-service-client 0.1.0", + "dns-service-client", "dpd-client", "dropshot", "expectorate", @@ -5047,7 +4592,7 @@ dependencies = [ "hubtools", "hyper", "hyper-rustls", - "internal-dns 0.1.0", + "internal-dns", "ipnetwork", "itertools 0.11.0", "lazy_static", @@ -5063,8 +4608,8 @@ dependencies = [ "nexus-test-utils-macros", "nexus-types", "num-integer", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", + "omicron-common", + "omicron-passwords", "omicron-rpaths", "omicron-sled-agent", "omicron-test-utils", @@ -5074,12 +4619,12 @@ dependencies = [ "openapiv3 1.0.3", "openssl", "oxide-client", - "oximeter 0.1.0", + "oximeter", "oximeter-client", "oximeter-db", "oximeter-instruments", - "oximeter-producer 0.1.0", - "parse-display 0.8.2", + "oximeter-producer", + "parse-display", "paste", "pem 1.1.1", "petgraph", @@ -5137,15 +4682,15 @@ dependencies = [ "gateway-messages", "gateway-test-utils", "humantime", - "internal-dns 0.1.0", + "internal-dns", "ipnetwork", - "nexus-client 0.1.0", + "nexus-client", "nexus-db-model", "nexus-db-queries", "nexus-test-utils", "nexus-test-utils-macros", "nexus-types", - "omicron-common 0.1.0", + "omicron-common", "omicron-nexus", "omicron-rpaths", "omicron-test-utils", @@ -5214,19 +4759,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "omicron-passwords" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "argon2", - "rand 0.8.5", - "schemars", - "serde", - "serde_with", - "thiserror", -] - [[package]] name = "omicron-rpaths" version = "0.1.0" @@ -5252,11 +4784,10 @@ dependencies = [ "chrono", "clap 4.4.3", "crucible-agent-client", - "crucible-client-types", "ddm-admin-client", "derive_more", "dns-server", - "dns-service-client 0.1.0", + "dns-service-client", "dpd-client", "dropshot", "expectorate", @@ -5268,27 +4799,27 @@ dependencies = [ "hyper", "hyper-staticfile", "illumos-utils", - "internal-dns 0.1.0", + "internal-dns", "ipnetwork", "itertools 0.11.0", "key-manager", "libc", "macaddr", "mg-admin-client", - "nexus-client 0.1.0", - "omicron-common 0.1.0", + "nexus-client", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "once_cell", "openapi-lint", "openapiv3 1.0.3", "opte-ioctl", - "oximeter 0.1.0", + "oximeter", "oximeter-instruments", - "oximeter-producer 0.1.0", + "oximeter-producer", "pretty_assertions", "propolis-client", - "propolis-server", + "propolis-mock-server", "rand 0.8.5", "rcgen", "reqwest", @@ -5335,7 +4866,7 @@ dependencies = [ "hex", "http", "libc", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "pem 1.1.1", "rcgen", @@ -5362,12 +4893,10 @@ dependencies = [ "bit-vec", "bitflags 1.3.2", "bitflags 2.4.0", - "bitvec", "bstr 0.2.17", "bstr 1.6.0", "byteorder", "bytes", - "cc", "chrono", "cipher", "clap 4.4.3", @@ -5393,7 +4922,6 @@ dependencies = [ "generic-array", "getrandom 0.2.10", "hashbrown 0.13.2", - "hashbrown 0.14.2", "hex", "hyper", "hyper-rustls", @@ -5413,6 +4941,7 @@ dependencies = [ "num-iter", "num-traits", "once_cell", + "openapiv3 2.0.0-rc.1", "petgraph", "postgres-types", "ppv-lite86", @@ -5422,7 +4951,6 @@ dependencies = [ "rand_chacha 0.3.1", "regex", "regex-automata 0.4.3", - "regex-syntax 0.6.29", "regex-syntax 0.8.2", "reqwest", "ring 0.16.20", @@ -5453,7 +4981,6 @@ dependencies = [ "trust-dns-proto", "unicode-bidi", "unicode-normalization", - "unicode-xid", "usdt", "uuid", "yasna", @@ -5706,9 +5233,9 @@ dependencies = [ "bytes", "chrono", "num", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", - "oximeter-macro-impl 0.1.0", + "oximeter-macro-impl", "rstest", "schemars", "serde", @@ -5718,29 +5245,13 @@ dependencies = [ "uuid", ] -[[package]] -name = "oximeter" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "bytes", - "chrono", - "num-traits", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter-macro-impl 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "schemars", - "serde", - "thiserror", - "uuid", -] - [[package]] name = "oximeter-client" version = "0.1.0" dependencies = [ "chrono", "futures", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "progenitor", "reqwest", @@ -5761,15 +5272,15 @@ dependencies = [ "expectorate", "futures", "hyper", - "internal-dns 0.1.0", - "nexus-client 0.1.0", + "internal-dns", + "nexus-client", "nexus-types", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "openapi-lint", "openapiv3 1.0.3", - "oximeter 0.1.0", + "oximeter", "oximeter-client", "oximeter-db", "rand 0.8.5", @@ -5804,10 +5315,10 @@ dependencies = [ "expectorate", "highway", "itertools 0.11.0", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", - "oximeter 0.1.0", + "oximeter", "regex", "reqwest", "schemars", @@ -5836,7 +5347,7 @@ dependencies = [ "http", "kstat-rs", "omicron-workspace-hack", - "oximeter 0.1.0", + "oximeter", "rand 0.8.5", "slog", "slog-async", @@ -5856,16 +5367,6 @@ dependencies = [ "syn 2.0.32", ] -[[package]] -name = "oximeter-macro-impl" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - [[package]] name = "oximeter-producer" version = "0.1.0" @@ -5874,30 +5375,10 @@ dependencies = [ "chrono", "clap 4.4.3", "dropshot", - "nexus-client 0.1.0", - "omicron-common 0.1.0", + "nexus-client", + "omicron-common", "omicron-workspace-hack", - "oximeter 0.1.0", - "schemars", - "serde", - "slog", - "slog-dtrace", - "thiserror", - "tokio", - "uuid", -] - -[[package]] -name = "oximeter-producer" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "chrono", - "dropshot", - "nexus-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "reqwest", + "oximeter", "schemars", "serde", "slog", @@ -5988,17 +5469,6 @@ dependencies = [ "windows-targets 0.48.5", ] -[[package]] -name = "parse-display" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6b32f6c8212838b74c0f5ba412194e88897923020810d9bec72d3594c2588d" -dependencies = [ - "once_cell", - "parse-display-derive 0.7.0", - "regex", -] - [[package]] name = "parse-display" version = "0.8.2" @@ -6006,23 +5476,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6509d08722b53e8dafe97f2027b22ccbe3a5db83cb352931e9716b0aa44bc5c" dependencies = [ "once_cell", - "parse-display-derive 0.8.2", - "regex", -] - -[[package]] -name = "parse-display-derive" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6ec9ab2477935d04fcdf7c51c9ee94a1be988938886de3239aed40980b7180" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", + "parse-display-derive", "regex", - "regex-syntax 0.6.29", - "structmeta 0.1.6", - "syn 1.0.109", ] [[package]] @@ -6036,7 +5491,7 @@ dependencies = [ "quote", "regex", "regex-syntax 0.7.5", - "structmeta 0.2.0", + "structmeta", "syn 2.0.32", ] @@ -6577,8 +6032,8 @@ dependencies = [ [[package]] name = "progenitor" -version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor?branch=main#5c941c0b41b0235031f3ade33a9c119945f1fd51" +version = "0.4.0" +source = "git+https://github.com/oxidecomputer/progenitor?branch=main#9339b57628e1e76b1d7131ef93a6c0db2ab0a762" dependencies = [ "progenitor-client", "progenitor-impl", @@ -6588,8 +6043,8 @@ dependencies = [ [[package]] name = "progenitor-client" -version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor?branch=main#5c941c0b41b0235031f3ade33a9c119945f1fd51" +version = "0.4.0" +source = "git+https://github.com/oxidecomputer/progenitor?branch=main#9339b57628e1e76b1d7131ef93a6c0db2ab0a762" dependencies = [ "bytes", "futures-core", @@ -6602,14 +6057,14 @@ dependencies = [ [[package]] name = "progenitor-impl" -version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor?branch=main#5c941c0b41b0235031f3ade33a9c119945f1fd51" +version = "0.4.0" +source = "git+https://github.com/oxidecomputer/progenitor?branch=main#9339b57628e1e76b1d7131ef93a6c0db2ab0a762" dependencies = [ "getopts", "heck 0.4.1", "http", "indexmap 2.1.0", - "openapiv3 1.0.3", + "openapiv3 2.0.0-rc.1", "proc-macro2", "quote", "regex", @@ -6624,10 +6079,10 @@ dependencies = [ [[package]] name = "progenitor-macro" -version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor?branch=main#5c941c0b41b0235031f3ade33a9c119945f1fd51" +version = "0.4.0" +source = "git+https://github.com/oxidecomputer/progenitor?branch=main#9339b57628e1e76b1d7131ef93a6c0db2ab0a762" dependencies = [ - "openapiv3 1.0.3", + "openapiv3 2.0.0-rc.1", "proc-macro2", "progenitor-impl", "quote", @@ -6639,53 +6094,17 @@ dependencies = [ "syn 2.0.32", ] -[[package]] -name = "propolis" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "anyhow", - "bhyve_api", - "bitflags 2.4.0", - "bitstruct", - "byteorder", - "crucible", - "crucible-client-types", - "dladm", - "erased-serde", - "futures", - "lazy_static", - "libc", - "nexus-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "propolis_types", - "rfb", - "serde", - "serde_arrays", - "serde_json", - "slog", - "strum", - "thiserror", - "tokio", - "usdt", - "uuid", - "viona_api", -] - [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" dependencies = [ "async-trait", "base64 0.21.5", - "crucible-client-types", "futures", "progenitor", - "propolis_types", "rand 0.8.5", "reqwest", - "ring 0.16.20", "schemars", "serde", "serde_json", @@ -6697,73 +6116,39 @@ dependencies = [ ] [[package]] -name = "propolis-server" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +name = "propolis-mock-server" +version = "0.0.0" +source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" dependencies = [ "anyhow", - "async-trait", "atty", "base64 0.21.5", - "bit_field", - "bitvec", - "bytes", - "cfg-if 1.0.0", - "chrono", "clap 4.4.3", - "const_format", - "crucible-client-types", "dropshot", - "erased-serde", "futures", - "http", "hyper", - "internal-dns 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "lazy_static", - "nexus-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter-producer 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "propolis", - "propolis-client", - "propolis-server-config", - "rfb", - "ron 0.7.1", + "progenitor", + "propolis_types", + "rand 0.8.5", + "reqwest", "schemars", "serde", - "serde_derive", "serde_json", "slog", "slog-async", "slog-bunyan", "slog-dtrace", "slog-term", - "strum", "thiserror", "tokio", "tokio-tungstenite 0.20.1", - "tokio-util", - "toml 0.7.8", - "usdt", "uuid", ] -[[package]] -name = "propolis-server-config" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "cpuid_profile_config", - "serde", - "serde_derive", - "thiserror", - "toml 0.7.8", -] - [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" dependencies = [ "schemars", "serde", @@ -7156,9 +6541,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "base64 0.21.5", "bytes", @@ -7184,6 +6569,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", @@ -7208,21 +6594,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "rfb" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/rfb?rev=0cac8d9c25eb27acfa35df80f3b9d371de98ab3b#0cac8d9c25eb27acfa35df80f3b9d371de98ab3b" -dependencies = [ - "ascii", - "async-trait", - "bitflags 1.3.2", - "env_logger 0.9.3", - "futures", - "log", - "thiserror", - "tokio", -] - [[package]] name = "ring" version = "0.16.20" @@ -7252,23 +6623,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "ringbuffer" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6368f71f205ff9c33c076d170dd56ebf68e8161c733c0caa07a7a5509ed53" - -[[package]] -name = "ron" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" -dependencies = [ - "base64 0.13.1", - "bitflags 1.3.2", - "serde", -] - [[package]] name = "ron" version = "0.8.1" @@ -7355,20 +6709,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rusqlite" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" -dependencies = [ - "bitflags 2.4.0", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec 1.11.0", -] - [[package]] name = "russh" version = "0.39.0" @@ -7827,15 +7167,6 @@ dependencies = [ "smallvec 0.6.14", ] -[[package]] -name = "serde_arrays" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38636132857f68ec3d5f3eb121166d2af33cb55174c4d5ff645db6165cbef0fd" -dependencies = [ - "serde", -] - [[package]] name = "serde_derive" version = "1.0.192" @@ -8178,7 +7509,7 @@ dependencies = [ "async-trait", "chrono", "ipnetwork", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "progenitor", "regress", @@ -8202,7 +7533,7 @@ dependencies = [ "libc", "libefi-illumos", "macaddr", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "rand 0.8.5", @@ -8228,7 +7559,7 @@ dependencies = [ "glob", "illumos-utils", "key-manager", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "rand 0.8.5", @@ -8455,7 +7786,7 @@ dependencies = [ "futures", "gateway-messages", "hex", - "omicron-common 0.1.0", + "omicron-common", "omicron-gateway", "omicron-workspace-hack", "serde", @@ -8598,18 +7929,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "structmeta" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "104842d6278bf64aa9d2f182ba4bde31e8aec7a131d29b7f444bb9b344a09e2a" -dependencies = [ - "proc-macro2", - "quote", - "structmeta-derive 0.1.6", - "syn 1.0.109", -] - [[package]] name = "structmeta" version = "0.2.0" @@ -8618,21 +7937,10 @@ checksum = "78ad9e09554f0456d67a69c1584c9798ba733a5b50349a6c0d0948710523922d" dependencies = [ "proc-macro2", "quote", - "structmeta-derive 0.2.0", + "structmeta-derive", "syn 2.0.32", ] -[[package]] -name = "structmeta-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24420be405b590e2d746d83b01f09af673270cf80e9b003a5fa7b651c58c7d93" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "structmeta-derive" version = "0.2.0" @@ -8769,6 +8077,27 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tabled" version = "0.14.0" @@ -8892,7 +8221,7 @@ checksum = "b8361c808554228ad09bfed70f5c823caf8a3450b6881cc3a38eb57e8c08c1d9" dependencies = [ "proc-macro2", "quote", - "structmeta 0.2.0", + "structmeta", "syn 2.0.32", ] @@ -9075,7 +8404,7 @@ name = "tlvc-text" version = "0.3.0" source = "git+https://github.com/oxidecomputer/tlvc.git#e644a21a7ca973ed31499106ea926bd63ebccc6f" dependencies = [ - "ron 0.8.1", + "ron", "serde", "tlvc 0.3.1 (git+https://github.com/oxidecomputer/tlvc.git)", "zerocopy 0.6.4", @@ -9492,7 +8821,7 @@ dependencies = [ "datatest-stable", "fs-err", "humantime", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "predicates 3.0.4", @@ -9521,7 +8850,7 @@ dependencies = [ "hex", "hubtools", "itertools 0.11.0", - "omicron-common 0.1.0", + "omicron-common", "omicron-test-utils", "omicron-workspace-hack", "rand 0.8.5", @@ -9586,17 +8915,6 @@ dependencies = [ "utf-8", ] -[[package]] -name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if 0.1.10", - "rand 0.8.5", - "static_assertions", -] - [[package]] name = "typenum" version = "1.16.0" @@ -9605,8 +8923,8 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "typify" -version = "0.0.13" -source = "git+https://github.com/oxidecomputer/typify#de16c4238a2b34400d0fece086a6469951c3236b" +version = "0.0.14" +source = "git+https://github.com/oxidecomputer/typify#c9d6453fc3cf69726d539925b838b267f886cb53" dependencies = [ "typify-impl", "typify-macro", @@ -9614,8 +8932,8 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.0.13" -source = "git+https://github.com/oxidecomputer/typify#de16c4238a2b34400d0fece086a6469951c3236b" +version = "0.0.14" +source = "git+https://github.com/oxidecomputer/typify#c9d6453fc3cf69726d539925b838b267f886cb53" dependencies = [ "heck 0.4.1", "log", @@ -9631,8 +8949,8 @@ dependencies = [ [[package]] name = "typify-macro" -version = "0.0.13" -source = "git+https://github.com/oxidecomputer/typify#de16c4238a2b34400d0fece086a6469951c3236b" +version = "0.0.14" +source = "git+https://github.com/oxidecomputer/typify#c9d6453fc3cf69726d539925b838b267f886cb53" dependencies = [ "proc-macro2", "quote", @@ -9880,42 +9198,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" -[[package]] -name = "vergen" -version = "8.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbc5ad0d9d26b2c49a5ab7da76c3e79d3ee37e7821799f8223fcb8f2f391a2e7" -dependencies = [ - "anyhow", - "git2", - "rustc_version 0.4.0", - "rustversion", - "time", -] - [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "viona_api" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "libc", - "viona_api_sys", -] - -[[package]] -name = "viona_api_sys" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "libc", -] - [[package]] name = "vsss-rs" version = "3.3.1" @@ -10135,8 +9423,8 @@ dependencies = [ "indexmap 2.1.0", "indicatif", "itertools 0.11.0", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", + "omicron-common", + "omicron-passwords", "omicron-workspace-hack", "once_cell", "owo-colors", @@ -10172,7 +9460,7 @@ version = "0.1.0" dependencies = [ "anyhow", "gateway-client", - "omicron-common 0.1.0", + "omicron-common", "omicron-workspace-hack", "schemars", "serde", @@ -10237,13 +9525,13 @@ dependencies = [ "installinator-artifact-client", "installinator-artifactd", "installinator-common", - "internal-dns 0.1.0", + "internal-dns", "ipnetwork", "itertools 0.11.0", "maplit", "omicron-certificates", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", + "omicron-common", + "omicron-passwords", "omicron-test-utils", "omicron-workspace-hack", "openapi-lint", diff --git a/Cargo.toml b/Cargo.toml index dfc6fe9c76..c51ac069a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,7 +170,6 @@ criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } -crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } curve25519-dalek = "4" @@ -291,9 +290,9 @@ pretty-hex = "0.3.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2", features = [ "generated-migration" ] } -propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2", default-features = false, features = ["mock-only"] } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" } +propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" } proptest = "1.3.1" quote = "1.0" rand = "0.8.5" @@ -547,9 +546,9 @@ opt-level = 3 #steno = { path = "../steno" } #[patch."https://github.com/oxidecomputer/propolis"] #propolis-client = { path = "../propolis/lib/propolis-client" } +#propolis-mock-server = { path = "../propolis/bin/mock-server" } #[patch."https://github.com/oxidecomputer/crucible"] #crucible-agent-client = { path = "../crucible/agent-client" } -#crucible-client-types = { path = "../crucible/crucible-client-types" } #crucible-pantry-client = { path = "../crucible/pantry-client" } #crucible-smf = { path = "../crucible/smf" } #[patch.crates-io] diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 46148304f8..61e61709e1 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -19,7 +19,6 @@ cfg-if.workspace = true chrono.workspace = true clap.workspace = true # Only used by the simulated sled agent. -crucible-client-types.workspace = true crucible-agent-client.workspace = true ddm-admin-client.workspace = true derive_more.workspace = true @@ -47,8 +46,8 @@ once_cell.workspace = true oximeter.workspace = true oximeter-instruments.workspace = true oximeter-producer.workspace = true -propolis-client = { workspace = true, features = [ "generated-migration" ] } -propolis-server.workspace = true # Only used by the simulated sled agent +propolis-client.workspace = true +propolis-mock-server.workspace = true # Only used by the simulated sled agent rand = { workspace = true, features = ["getrandom"] } reqwest = { workspace = true, features = ["rustls-tls", "stream"] } schemars = { workspace = true, features = [ "chrono", "uuid1" ] } diff --git a/sled-agent/src/common/disk.rs b/sled-agent/src/common/disk.rs index 18160950d3..57868937d0 100644 --- a/sled-agent/src/common/disk.rs +++ b/sled-agent/src/common/disk.rs @@ -9,7 +9,7 @@ use chrono::Utc; use omicron_common::api::external::DiskState; use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::DiskRuntimeState; -use propolis_client::api::DiskAttachmentState as PropolisDiskState; +use propolis_client::types::DiskAttachmentState as PropolisDiskState; use uuid::Uuid; /// Action to be taken on behalf of state transition. diff --git a/sled-agent/src/common/instance.rs b/sled-agent/src/common/instance.rs index 9e285840e0..d7ee8982e0 100644 --- a/sled-agent/src/common/instance.rs +++ b/sled-agent/src/common/instance.rs @@ -10,8 +10,9 @@ use omicron_common::api::external::InstanceState as ApiInstanceState; use omicron_common::api::internal::nexus::{ InstanceRuntimeState, SledInstanceState, VmmRuntimeState, }; -use propolis_client::api::{ +use propolis_client::types::{ InstanceState as PropolisApiState, InstanceStateMonitorResponse, + MigrationState, }; use uuid::Uuid; @@ -36,7 +37,7 @@ impl From for PropolisInstanceState { impl From for ApiInstanceState { fn from(value: PropolisInstanceState) -> Self { - use propolis_client::api::InstanceState as State; + use propolis_client::types::InstanceState as State; match value.0 { // Nexus uses the VMM state as the externally-visible instance state // when an instance has an active VMM. A Propolis that is "creating" @@ -119,7 +120,6 @@ impl ObservedPropolisState { (Some(this_id), Some(propolis_migration)) if this_id == propolis_migration.migration_id => { - use propolis_client::api::MigrationState; match propolis_migration.state { MigrationState::Finish => { ObservedMigrationStatus::Succeeded @@ -510,7 +510,7 @@ mod test { use chrono::Utc; use omicron_common::api::external::Generation; use omicron_common::api::internal::nexus::InstanceRuntimeState; - use propolis_client::api::InstanceState as Observed; + use propolis_client::types::InstanceState as Observed; use uuid::Uuid; fn make_instance() -> InstanceStates { diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index f030078761..a6f022f5f2 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -190,7 +190,7 @@ struct InstanceInner { log: Logger, // Properties visible to Propolis - properties: propolis_client::api::InstanceProperties, + properties: propolis_client::types::InstanceProperties, // The ID of the Propolis server (and zone) running this instance propolis_id: Uuid, @@ -213,8 +213,7 @@ struct InstanceInner { dhcp_config: DhcpCfg, // Disk related properties - // TODO: replace `propolis_client::handmade::*` with properly-modeled local types - requested_disks: Vec, + requested_disks: Vec, cloud_init_bytes: Option, // Internal State management @@ -379,7 +378,7 @@ impl InstanceInner { /// Sends an instance state PUT request to this instance's Propolis. async fn propolis_state_put( &self, - request: propolis_client::api::InstanceStateRequested, + request: propolis_client::types::InstanceStateRequested, ) -> Result<(), Error> { let res = self .running_state @@ -409,11 +408,11 @@ impl InstanceInner { ) -> Result<(), Error> { let nics = running_zone .opte_ports() - .map(|port| propolis_client::api::NetworkInterfaceRequest { + .map(|port| propolis_client::types::NetworkInterfaceRequest { // TODO-correctness: Remove `.vnic()` call when we use the port // directly. name: port.vnic_name().to_string(), - slot: propolis_client::api::Slot(port.slot()), + slot: propolis_client::types::Slot(port.slot()), }) .collect(); @@ -423,7 +422,7 @@ impl InstanceInner { self.state.instance().migration_id.ok_or_else(|| { Error::Migration(anyhow!("Missing Migration UUID")) })?; - Some(propolis_client::api::InstanceMigrateInitiateRequest { + Some(propolis_client::types::InstanceMigrateInitiateRequest { src_addr: params.src_propolis_addr.to_string(), src_uuid: params.src_propolis_id, migration_id, @@ -432,7 +431,7 @@ impl InstanceInner { None => None, }; - let request = propolis_client::api::InstanceEnsureRequest { + let request = propolis_client::types::InstanceEnsureRequest { properties: self.properties.clone(), nics, disks: self @@ -648,7 +647,7 @@ impl Instance { let instance = InstanceInner { log: log.new(o!("instance_id" => id.to_string())), // NOTE: Mostly lies. - properties: propolis_client::api::InstanceProperties { + properties: propolis_client::types::InstanceProperties { id, name: hardware.properties.hostname.clone(), description: "Test description".to_string(), @@ -789,7 +788,7 @@ impl Instance { &self, state: crate::params::InstanceStateRequested, ) -> Result { - use propolis_client::api::InstanceStateRequested as PropolisRequest; + use propolis_client::types::InstanceStateRequested as PropolisRequest; let mut inner = self.inner.lock().await; let (propolis_state, next_published) = match state { InstanceStateRequested::MigrationTarget(migration_params) => { @@ -1036,7 +1035,9 @@ impl Instance { // known to Propolis. let response = client .instance_state_monitor() - .body(propolis_client::api::InstanceStateMonitorRequest { gen }) + .body(propolis_client::types::InstanceStateMonitorRequest { + gen, + }) .send() .await? .into_inner(); diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 67d5f049e7..b22bd84975 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -71,8 +71,8 @@ pub struct InstanceHardware { pub external_ips: Vec, pub firewall_rules: Vec, pub dhcp_config: DhcpConfig, - // TODO: replace `propolis_client::handmade::*` with locally-modeled request type - pub disks: Vec, + // TODO: replace `propolis_client::*` with locally-modeled request type + pub disks: Vec, pub cloud_init_bytes: Option, } diff --git a/sled-agent/src/sim/disk.rs b/sled-agent/src/sim/disk.rs index 2d2c18be25..0f08289b74 100644 --- a/sled-agent/src/sim/disk.rs +++ b/sled-agent/src/sim/disk.rs @@ -19,7 +19,7 @@ use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::ProducerEndpoint; use oximeter_producer::LogConfig; use oximeter_producer::Server as ProducerServer; -use propolis_client::api::DiskAttachmentState as PropolisDiskState; +use propolis_client::types::DiskAttachmentState as PropolisDiskState; use std::net::{Ipv6Addr, SocketAddr}; use std::sync::Arc; use std::time::Duration; diff --git a/sled-agent/src/sim/http_entrypoints_pantry.rs b/sled-agent/src/sim/http_entrypoints_pantry.rs index 46686a74e2..64b26a83a4 100644 --- a/sled-agent/src/sim/http_entrypoints_pantry.rs +++ b/sled-agent/src/sim/http_entrypoints_pantry.rs @@ -4,11 +4,11 @@ //! HTTP entrypoint functions for simulating the crucible pantry API. -use crucible_client_types::VolumeConstructionRequest; use dropshot::{ endpoint, ApiDescription, HttpError, HttpResponseDeleted, HttpResponseOk, HttpResponseUpdatedNoContent, Path as TypedPath, RequestContext, TypedBody, }; +use propolis_client::types::VolumeConstructionRequest; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::sync::Arc; diff --git a/sled-agent/src/sim/instance.rs b/sled-agent/src/sim/instance.rs index 397a1980a5..15ff83c969 100644 --- a/sled-agent/src/sim/instance.rs +++ b/sled-agent/src/sim/instance.rs @@ -19,9 +19,10 @@ use omicron_common::api::external::ResourceType; use omicron_common::api::internal::nexus::{ InstanceRuntimeState, SledInstanceState, }; -use propolis_client::api::InstanceMigrateStatusResponse as PropolisMigrateStatus; -use propolis_client::api::InstanceState as PropolisInstanceState; -use propolis_client::api::InstanceStateMonitorResponse; +use propolis_client::types::{ + InstanceMigrateStatusResponse as PropolisMigrateStatus, + InstanceState as PropolisInstanceState, InstanceStateMonitorResponse, +}; use std::collections::VecDeque; use std::sync::Arc; use std::sync::Mutex; @@ -131,11 +132,11 @@ impl SimInstanceInner { }); self.queue_migration_status(PropolisMigrateStatus { migration_id, - state: propolis_client::api::MigrationState::Sync, + state: propolis_client::types::MigrationState::Sync, }); self.queue_migration_status(PropolisMigrateStatus { migration_id, - state: propolis_client::api::MigrationState::Finish, + state: propolis_client::types::MigrationState::Finish, }); self.queue_propolis_state(PropolisInstanceState::Running); } diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index e4dac2f4b9..c06ae96f2e 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -35,15 +35,16 @@ use uuid::Uuid; use std::collections::HashMap; use std::str::FromStr; -use crucible_client_types::VolumeConstructionRequest; use dropshot::HttpServer; use illumos_utils::opte::params::{ DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost, }; use nexus_client::types::PhysicalDiskKind; use omicron_common::address::PROPOLIS_PORT; -use propolis_client::Client as PropolisClient; -use propolis_server::mock_server::Context as PropolisContext; +use propolis_client::{ + types::VolumeConstructionRequest, Client as PropolisClient, +}; +use propolis_mock_server::Context as PropolisContext; /// Simulates management of the control plane on a sled /// @@ -70,13 +71,14 @@ pub struct SledAgent { } fn extract_targets_from_volume_construction_request( - vec: &mut Vec, vcr: &VolumeConstructionRequest, -) { +) -> Result, std::net::AddrParseError> { // A snapshot is simply a flush with an extra parameter, and flushes are // only sent to sub volumes, not the read only parent. Flushes are only // processed by regions, so extract each region that would be affected by a // flush. + + let mut res = vec![]; match vcr { VolumeConstructionRequest::Volume { id: _, @@ -85,9 +87,9 @@ fn extract_targets_from_volume_construction_request( read_only_parent: _, } => { for sub_volume in sub_volumes.iter() { - extract_targets_from_volume_construction_request( - vec, sub_volume, - ); + res.extend(extract_targets_from_volume_construction_request( + sub_volume, + )?); } } @@ -103,7 +105,7 @@ fn extract_targets_from_volume_construction_request( gen: _, } => { for target in &opts.target { - vec.push(*target); + res.push(SocketAddr::from_str(target)?); } } @@ -111,6 +113,7 @@ fn extract_targets_from_volume_construction_request( // noop } } + Ok(res) } impl SledAgent { @@ -171,23 +174,19 @@ impl SledAgent { volume_construction_request: &VolumeConstructionRequest, ) -> Result<(), Error> { let disk_id = match volume_construction_request { - VolumeConstructionRequest::Volume { - id, - block_size: _, - sub_volumes: _, - read_only_parent: _, - } => id, + VolumeConstructionRequest::Volume { id, .. } => id, _ => { panic!("root of volume construction request not a volume!"); } }; - let mut targets = Vec::new(); - extract_targets_from_volume_construction_request( - &mut targets, + let targets = extract_targets_from_volume_construction_request( &volume_construction_request, - ); + ) + .map_err(|e| { + Error::invalid_request(&format!("bad socketaddr: {e:?}")) + })?; let mut region_ids = Vec::new(); @@ -640,11 +639,10 @@ impl SledAgent { ..Default::default() }; let propolis_log = log.new(o!("component" => "propolis-server-mock")); - let private = - Arc::new(PropolisContext::new(Default::default(), propolis_log)); + let private = Arc::new(PropolisContext::new(propolis_log)); info!(log, "Starting mock propolis-server..."); let dropshot_log = log.new(o!("component" => "dropshot")); - let mock_api = propolis_server::mock_server::api(); + let mock_api = propolis_mock_server::api(); let srv = dropshot::HttpServerStarter::new( &dropshot_config, diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 89b1f59c7a..2528a258d7 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -16,13 +16,13 @@ use chrono::prelude::*; use crucible_agent_client::types::{ CreateRegion, Region, RegionId, RunningSnapshot, Snapshot, State, }; -use crucible_client_types::VolumeConstructionRequest; use dropshot::HandlerTaskMode; use dropshot::HttpError; use futures::lock::Mutex; use nexus_client::types::{ ByteCount, PhysicalDiskKind, PhysicalDiskPutRequest, ZpoolPutRequest, }; +use propolis_client::types::VolumeConstructionRequest; use slog::Logger; use std::collections::HashMap; use std::collections::HashSet; diff --git a/tools/update_crucible.sh b/tools/update_crucible.sh index af834091ca..020a33927e 100755 --- a/tools/update_crucible.sh +++ b/tools/update_crucible.sh @@ -21,7 +21,6 @@ PACKAGES=( CRATES=( "crucible-agent-client" - "crucible-client-types" "crucible-pantry-client" "crucible-smf" ) diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index be9ebd7847..c95226b960 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -19,7 +19,6 @@ bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["serde"] } -bitvec = { version = "1.0.1" } bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" } bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" } byteorder = { version = "1.5.0" } @@ -48,8 +47,7 @@ futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] } -hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } +hashbrown = { version = "0.13.2" } hex = { version = "0.4.3", features = ["serde"] } hyper = { version = "0.14.27", features = ["full"] } indexmap = { version = "2.1.0", features = ["serde"] } @@ -66,17 +64,18 @@ num-bigint = { version = "0.4.4", features = ["rand"] } num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } +openapiv3 = { version = "2.0.0-rc.1", default-features = false, features = ["skip_serializing_defaults"] } petgraph = { version = "0.6.4", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.0.4" } proc-macro2 = { version = "1.0.69" } -rand = { version = "0.8.5", features = ["min_const_gen", "small_rng"] } -rand_chacha = { version = "0.3.1" } +rand = { version = "0.8.5" } +rand_chacha = { version = "0.3.1", default-features = false, features = ["std"] } regex = { version = "1.10.2" } regex-automata = { version = "0.4.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } -regex-syntax-c38e5c1d305a1b54 = { package = "regex-syntax", version = "0.8.2" } -reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } +regex-syntax = { version = "0.8.2" } +reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] } ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.20", features = ["serde"] } @@ -113,12 +112,10 @@ bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["serde"] } -bitvec = { version = "1.0.1" } bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" } bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" } byteorder = { version = "1.5.0" } bytes = { version = "1.5.0", features = ["serde"] } -cc = { version = "1.0.83", default-features = false, features = ["parallel"] } chrono = { version = "0.4.31", features = ["alloc", "serde"] } cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] } clap = { version = "4.4.3", features = ["derive", "env", "wrap_help"] } @@ -143,8 +140,7 @@ futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.2", features = ["raw"] } -hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } +hashbrown = { version = "0.13.2" } hex = { version = "0.4.3", features = ["serde"] } hyper = { version = "0.14.27", features = ["full"] } indexmap = { version = "2.1.0", features = ["serde"] } @@ -161,18 +157,18 @@ num-bigint = { version = "0.4.4", features = ["rand"] } num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } +openapiv3 = { version = "2.0.0-rc.1", default-features = false, features = ["skip_serializing_defaults"] } petgraph = { version = "0.6.4", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.0.4" } proc-macro2 = { version = "1.0.69" } -rand = { version = "0.8.5", features = ["min_const_gen", "small_rng"] } -rand_chacha = { version = "0.3.1" } +rand = { version = "0.8.5" } +rand_chacha = { version = "0.3.1", default-features = false, features = ["std"] } regex = { version = "1.10.2" } regex-automata = { version = "0.4.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } -regex-syntax-3b31131e45eafb45 = { package = "regex-syntax", version = "0.6.29" } -regex-syntax-c38e5c1d305a1b54 = { package = "regex-syntax", version = "0.8.2" } -reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } +regex-syntax = { version = "0.8.2" } +reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] } ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.20", features = ["serde"] } @@ -198,7 +194,6 @@ tracing = { version = "0.1.37", features = ["log"] } trust-dns-proto = { version = "0.22.0" } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } -unicode-xid = { version = "0.2.4" } usdt = { version = "0.3.5" } uuid = { version = "1.5.0", features = ["serde", "v4"] } yasna = { version = "0.5.2", features = ["bit-vec", "num-bigint", "std", "time"] } From 9c33a6268dc5c3ddfab4696b47f90260878e821c Mon Sep 17 00:00:00 2001 From: bnaecker Date: Wed, 15 Nov 2023 11:22:35 -0800 Subject: [PATCH 03/11] Send oximeter all existing producer assignments on registration (#4500) - Fixes #4498 - When `oximeter` registers with `nexus`, we send it any producer assignments so that it can continue collecting after a restart. The list of assignments is paginated from the database, but we previously only sent the first page. This ensures we send all records, and adds a regression test. --- nexus/src/app/oximeter.rs | 54 ++++++++------ nexus/tests/integration_tests/oximeter.rs | 90 +++++++++++++++++++++++ 2 files changed, 122 insertions(+), 22 deletions(-) diff --git a/nexus/src/app/oximeter.rs b/nexus/src/app/oximeter.rs index 03f833b087..7dfa2fb68b 100644 --- a/nexus/src/app/oximeter.rs +++ b/nexus/src/app/oximeter.rs @@ -87,32 +87,43 @@ impl super::Nexus { "address" => oximeter_info.address, ); - // Regardless, notify the collector of any assigned metric producers. This should be empty - // if this Oximeter collector is registering for the first time, but may not be if the - // service is re-registering after failure. - let pagparams = DataPageParams { - marker: None, - direction: PaginationOrder::Ascending, - limit: std::num::NonZeroU32::new(100).unwrap(), - }; - let producers = self - .db_datastore - .producers_list_by_oximeter_id( - oximeter_info.collector_id, - &pagparams, - ) - .await?; - if !producers.is_empty() { + // Regardless, notify the collector of any assigned metric producers. + // + // This should be empty if this Oximeter collector is registering for + // the first time, but may not be if the service is re-registering after + // failure. + let client = self.build_oximeter_client( + &oximeter_info.collector_id, + oximeter_info.address, + ); + let mut last_producer_id = None; + loop { + let pagparams = DataPageParams { + marker: last_producer_id.as_ref(), + direction: PaginationOrder::Ascending, + limit: std::num::NonZeroU32::new(100).unwrap(), + }; + let producers = self + .db_datastore + .producers_list_by_oximeter_id( + oximeter_info.collector_id, + &pagparams, + ) + .await?; + if producers.is_empty() { + return Ok(()); + } debug!( self.log, - "registered oximeter collector that is already assigned producers, re-assigning them to the collector"; + "re-assigning existing metric producers to a collector"; "n_producers" => producers.len(), "collector_id" => ?oximeter_info.collector_id, ); - let client = self.build_oximeter_client( - &oximeter_info.collector_id, - oximeter_info.address, - ); + // Be sure to continue paginating from the last producer. + // + // Safety: We check just above if the list is empty, so there is a + // last element. + last_producer_id.replace(producers.last().unwrap().id()); for producer in producers.into_iter() { let producer_info = oximeter_client::types::ProducerEndpoint { id: producer.id(), @@ -132,7 +143,6 @@ impl super::Nexus { .map_err(Error::from)?; } } - Ok(()) } /// Register as a metric producer with the oximeter metric collection server. diff --git a/nexus/tests/integration_tests/oximeter.rs b/nexus/tests/integration_tests/oximeter.rs index 2cda594e18..65aaa18642 100644 --- a/nexus/tests/integration_tests/oximeter.rs +++ b/nexus/tests/integration_tests/oximeter.rs @@ -4,11 +4,17 @@ //! Integration tests for oximeter collectors and producers. +use dropshot::Method; +use http::StatusCode; use nexus_test_interface::NexusServer; use nexus_test_utils_macros::nexus_test; +use omicron_common::api::internal::nexus::ProducerEndpoint; use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; use oximeter_db::DbWrite; +use std::collections::BTreeSet; use std::net; +use std::net::Ipv6Addr; +use std::net::SocketAddr; use std::time::Duration; use uuid::Uuid; @@ -332,3 +338,87 @@ async fn test_oximeter_reregistration() { ); context.teardown().await; } + +// A regression test for https://github.com/oxidecomputer/omicron/issues/4498 +#[tokio::test] +async fn test_oximeter_collector_reregistration_gets_all_assignments() { + let mut context = nexus_test_utils::test_setup::( + "test_oximeter_collector_reregistration_gets_all_assignments", + ) + .await; + let oximeter_id = nexus_test_utils::OXIMETER_UUID.parse().unwrap(); + + // Create a bunch of producer records. + // + // Note that the actual count is arbitrary, but it should be larger than the + // internal pagination limit used in `Nexus::upsert_oximeter_collector()`, + // which is currently 100. + const N_PRODUCERS: usize = 150; + let mut ids = BTreeSet::new(); + for _ in 0..N_PRODUCERS { + let id = Uuid::new_v4(); + ids.insert(id); + let info = ProducerEndpoint { + id, + address: SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 12345), + base_route: String::from("/collect"), + interval: Duration::from_secs(1), + }; + context + .internal_client + .make_request( + Method::POST, + "/metrics/producers", + Some(&info), + StatusCode::NO_CONTENT, + ) + .await + .expect("failed to register test producer"); + } + + // Check that `oximeter` has these registered. + let producers = + context.oximeter.list_producers(None, N_PRODUCERS * 2).await; + let actual_ids: BTreeSet<_> = + producers.iter().map(|info| info.id).collect(); + + // There is an additional producer that's created as part of the normal test + // setup, so we'll check that all of the new producers exist, and that + // there's exactly 1 additional one. + assert!( + ids.is_subset(&actual_ids), + "oximeter did not get the right set of producers" + ); + assert_eq!( + ids.len(), + actual_ids.len() - 1, + "oximeter did not get the right set of producers" + ); + + // Drop and restart oximeter, which should result in the exact same set of + // producers again. + drop(context.oximeter); + context.oximeter = nexus_test_utils::start_oximeter( + context.logctx.log.new(o!("component" => "oximeter")), + context.server.get_http_server_internal_address().await, + context.clickhouse.port(), + oximeter_id, + ) + .await + .expect("failed to restart oximeter"); + + let producers = + context.oximeter.list_producers(None, N_PRODUCERS * 2).await; + let actual_ids: BTreeSet<_> = + producers.iter().map(|info| info.id).collect(); + assert!( + ids.is_subset(&actual_ids), + "oximeter did not get the right set of producers after re-registering" + ); + assert_eq!( + ids.len(), + actual_ids.len() - 1, + "oximeter did not get the right set of producers after re-registering" + ); + context.teardown().await; +} From cc5c40e03f9c52e54cb56c228b21d4cbcad49d4f Mon Sep 17 00:00:00 2001 From: Patrick Mooney Date: Wed, 15 Nov 2023 22:23:29 -0600 Subject: [PATCH 04/11] Pare down nexus-client dependencies Adding sled-hardware and sled-storage as dependencies to nexus-client bring in a large portion of unrelated crates, all in service of some (albeit convenient) type conversion routines. For the sake of nexus-client consumers outside of omicron, we can manually implement those few conversions in sled-agent where they are consumed. --- Cargo.lock | 2 -- clients/nexus-client/Cargo.toml | 2 -- clients/nexus-client/src/lib.rs | 33 ------------------- sled-agent/src/nexus.rs | 48 ++++++++++++++++++++++++++++ sled-agent/src/rack_setup/service.rs | 4 +-- sled-agent/src/sled_agent.rs | 6 ++-- sled-agent/src/storage_monitor.rs | 6 ++-- 7 files changed, 55 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b3007b86e..f98f7c06ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3934,8 +3934,6 @@ dependencies = [ "schemars", "serde", "serde_json", - "sled-hardware", - "sled-storage", "slog", "uuid", ] diff --git a/clients/nexus-client/Cargo.toml b/clients/nexus-client/Cargo.toml index 239cb77789..2734142f9f 100644 --- a/clients/nexus-client/Cargo.toml +++ b/clients/nexus-client/Cargo.toml @@ -10,8 +10,6 @@ futures.workspace = true ipnetwork.workspace = true omicron-common.workspace = true omicron-passwords.workspace = true -sled-hardware.workspace = true -sled-storage.workspace = true progenitor.workspace = true regress.workspace = true reqwest = { workspace = true, features = ["rustls-tls", "stream"] } diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs index 9f81492d10..23ceb114fc 100644 --- a/clients/nexus-client/src/lib.rs +++ b/clients/nexus-client/src/lib.rs @@ -388,36 +388,3 @@ impl From } } } - -impl From for types::PhysicalDiskKind { - fn from(value: sled_hardware::DiskVariant) -> Self { - match value { - sled_hardware::DiskVariant::U2 => types::PhysicalDiskKind::U2, - sled_hardware::DiskVariant::M2 => types::PhysicalDiskKind::M2, - } - } -} - -impl From for types::Baseboard { - fn from(b: sled_hardware::Baseboard) -> types::Baseboard { - types::Baseboard { - serial_number: b.identifier().to_string(), - part_number: b.model().to_string(), - revision: b.revision(), - } - } -} - -impl From for types::DatasetKind { - fn from(k: sled_storage::dataset::DatasetKind) -> Self { - use sled_storage::dataset::DatasetKind::*; - match k { - CockroachDb => Self::Cockroach, - Crucible => Self::Crucible, - Clickhouse => Self::Clickhouse, - ClickhouseKeeper => Self::ClickhouseKeeper, - ExternalDns => Self::ExternalDns, - InternalDns => Self::InternalDns, - } - } -} diff --git a/sled-agent/src/nexus.rs b/sled-agent/src/nexus.rs index 2af6fa0023..cc715f4010 100644 --- a/sled-agent/src/nexus.rs +++ b/sled-agent/src/nexus.rs @@ -154,3 +154,51 @@ fn d2n_record( } } } + +// Although it is a bit awkward to define these conversions here, it frees us +// from depending on sled_storage/sled_hardware in the nexus_client crate. + +pub(crate) trait ConvertInto: Sized { + fn convert(self) -> T; +} + +impl ConvertInto + for sled_hardware::DiskVariant +{ + fn convert(self) -> nexus_client::types::PhysicalDiskKind { + use nexus_client::types::PhysicalDiskKind; + + match self { + sled_hardware::DiskVariant::U2 => PhysicalDiskKind::U2, + sled_hardware::DiskVariant::M2 => PhysicalDiskKind::M2, + } + } +} + +impl ConvertInto for sled_hardware::Baseboard { + fn convert(self) -> nexus_client::types::Baseboard { + nexus_client::types::Baseboard { + serial_number: self.identifier().to_string(), + part_number: self.model().to_string(), + revision: self.revision(), + } + } +} + +impl ConvertInto + for sled_storage::dataset::DatasetKind +{ + fn convert(self) -> nexus_client::types::DatasetKind { + use nexus_client::types::DatasetKind; + use sled_storage::dataset::DatasetKind::*; + + match self { + CockroachDb => DatasetKind::Cockroach, + Crucible => DatasetKind::Crucible, + Clickhouse => DatasetKind::Clickhouse, + ClickhouseKeeper => DatasetKind::ClickhouseKeeper, + ExternalDns => DatasetKind::ExternalDns, + InternalDns => DatasetKind::InternalDns, + } + } +} diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 34d5e06cfe..7dcbfa7045 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -63,7 +63,7 @@ use crate::bootstrap::early_networking::{ use crate::bootstrap::params::BootstrapAddressDiscovery; use crate::bootstrap::params::StartSledAgentRequest; use crate::bootstrap::rss_handle::BootstrapAgentHandle; -use crate::nexus::d2n_params; +use crate::nexus::{d2n_params, ConvertInto}; use crate::params::{ AutonomousServiceOnlyError, ServiceType, ServiceZoneRequest, ServiceZoneService, TimeSync, ZoneType, @@ -564,7 +564,7 @@ impl ServiceInner { dataset_id: dataset.id, request: NexusTypes::DatasetPutRequest { address: dataset.service_address.to_string(), - kind: dataset.name.dataset().clone().into(), + kind: dataset.name.dataset().clone().convert(), }, }) } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index aec64a1349..9826a987d4 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -13,7 +13,7 @@ use crate::config::Config; use crate::instance_manager::{InstanceManager, ReservoirMode}; use crate::long_running_tasks::LongRunningTaskHandles; use crate::metrics::MetricsManager; -use crate::nexus::{NexusClientWithResolver, NexusRequestQueue}; +use crate::nexus::{ConvertInto, NexusClientWithResolver, NexusRequestQueue}; use crate::params::{ DiskStateRequested, InstanceHardware, InstanceMigrationSourceParams, InstancePutStateResponse, InstanceStateRequested, @@ -607,9 +607,7 @@ impl SledAgent { let nexus_client = self.inner.nexus_client.clone(); let sled_address = self.inner.sled_address(); let is_scrimlet = self.inner.hardware.is_scrimlet(); - let baseboard = nexus_client::types::Baseboard::from( - self.inner.hardware.baseboard(), - ); + let baseboard = self.inner.hardware.baseboard().convert(); let usable_hardware_threads = self.inner.hardware.online_processor_count(); let usable_physical_ram = diff --git a/sled-agent/src/storage_monitor.rs b/sled-agent/src/storage_monitor.rs index f552fdfd86..0c9b287396 100644 --- a/sled-agent/src/storage_monitor.rs +++ b/sled-agent/src/storage_monitor.rs @@ -7,7 +7,7 @@ //! code. use crate::dump_setup::DumpSetup; -use crate::nexus::NexusClientWithResolver; +use crate::nexus::{ConvertInto, NexusClientWithResolver}; use derive_more::From; use futures::stream::FuturesOrdered; use futures::FutureExt; @@ -338,7 +338,7 @@ fn compute_resource_diffs( model: disk_id.model.clone(), serial: disk_id.serial.clone(), vendor: disk_id.vendor.clone(), - variant: updated_disk.variant().into(), + variant: updated_disk.variant().convert(), }); } if pool != updated_pool { @@ -363,7 +363,7 @@ fn compute_resource_diffs( model: disk_id.model.clone(), serial: disk_id.serial.clone(), vendor: disk_id.vendor.clone(), - variant: updated_disk.variant().into(), + variant: updated_disk.variant().convert(), }); put_pool(disk_id, updated_pool); } From 9ef3f328a1331970492070d87a385f9f246ee251 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Thu, 16 Nov 2023 07:47:35 -0800 Subject: [PATCH 05/11] Update Crucible (51a3121) and Propolis (8ad2d4f) (#4499) Crucible changes: test-crudd can collect more info, test_up can be gentle (#997) Decrypt without holding the downstairs lock (#1021) Add raw file backend (#991) Don't hold the Downstairs lock while doing encryption (#1019) Antagonize the Crucible Agent (#1011) The Pantry should reject non-block sized writes (#1013) Propolis changes: make headroom for linux virtio/9p client impl (#565) Guarantee Tokio access for Entity methods --------- Co-authored-by: Alan Hanson --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 12 ++++++------ package-manifest.toml | 12 ++++++------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f98f7c06ba..58d0653728 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -447,7 +447,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" +source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" dependencies = [ "bhyve_api_sys", "libc", @@ -457,7 +457,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" +source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" dependencies = [ "libc", "strum", @@ -1270,7 +1270,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" +source = "git+https://github.com/oxidecomputer/crucible?rev=51a3121c8318fc7ac97d74f917ce1d37962e785f#51a3121c8318fc7ac97d74f917ce1d37962e785f" dependencies = [ "anyhow", "chrono", @@ -1286,7 +1286,7 @@ dependencies = [ [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" +source = "git+https://github.com/oxidecomputer/crucible?rev=51a3121c8318fc7ac97d74f917ce1d37962e785f#51a3121c8318fc7ac97d74f917ce1d37962e785f" dependencies = [ "anyhow", "chrono", @@ -1303,7 +1303,7 @@ dependencies = [ [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" +source = "git+https://github.com/oxidecomputer/crucible?rev=51a3121c8318fc7ac97d74f917ce1d37962e785f#51a3121c8318fc7ac97d74f917ce1d37962e785f" dependencies = [ "crucible-workspace-hack", "libc", @@ -6095,7 +6095,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" +source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" dependencies = [ "async-trait", "base64 0.21.5", @@ -6116,7 +6116,7 @@ dependencies = [ [[package]] name = "propolis-mock-server" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" +source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" dependencies = [ "anyhow", "atty", @@ -6146,7 +6146,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=5ed82315541271e2734746a9ca79e39f35c12283#5ed82315541271e2734746a9ca79e39f35c12283" +source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" dependencies = [ "schemars", "serde", diff --git a/Cargo.toml b/Cargo.toml index c51ac069a9..82bca496a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -169,9 +169,9 @@ cookie = "0.16" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "51a3121c8318fc7ac97d74f917ce1d37962e785f" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "51a3121c8318fc7ac97d74f917ce1d37962e785f" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "51a3121c8318fc7ac97d74f917ce1d37962e785f" } curve25519-dalek = "4" datatest-stable = "0.2.3" display-error-chain = "0.2.0" @@ -290,9 +290,9 @@ pretty-hex = "0.3.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" } -propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" } +propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" } proptest = "1.3.1" quote = "1.0" rand = "0.8.5" diff --git a/package-manifest.toml b/package-manifest.toml index 61c90a3e75..c65c5b6933 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -384,10 +384,10 @@ only_for_targets.image = "standard" # 3. Use source.type = "manual" instead of "prebuilt" source.type = "prebuilt" source.repo = "crucible" -source.commit = "da534e73380f3cc53ca0de073e1ea862ae32109b" +source.commit = "51a3121c8318fc7ac97d74f917ce1d37962e785f" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible.sha256.txt -source.sha256 = "572ac3b19e51b4e476266a62c2b7e06eff81c386cb48247c4b9f9b1e2ee81895" +source.sha256 = "897d0fd6c0b82db42256a63a13c228152e1117434afa2681f649b291e3c6f46d" output.type = "zone" [package.crucible-pantry] @@ -395,10 +395,10 @@ service_name = "crucible_pantry" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "da534e73380f3cc53ca0de073e1ea862ae32109b" +source.commit = "51a3121c8318fc7ac97d74f917ce1d37962e785f" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-pantry.sha256.txt -source.sha256 = "812269958e18f54d72bc10bb4fb81f26c084cf762da7fd98e63d58c689be9ad1" +source.sha256 = "fe545de7ac4f15454d7827927149c5f0fc68ce9545b4f1ef96aac9ac8039805a" output.type = "zone" # Refer to @@ -409,10 +409,10 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +source.commit = "54398875a2125227d13827d4236dce943c019b1c" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "aa1d9dc5c9117c100f9636901e8eec6679d7dfbf869c46b7f2873585f94a1b89" +source.sha256 = "01b8563db6626f90ee3fb6d97e7921b0a680373d843c1bea7ebf46fcea4f7b28" output.type = "zone" [package.mg-ddm-gz] From ca9d90a79c2a7a9ffdca12a705d007d7f28ce61f Mon Sep 17 00:00:00 2001 From: "oxide-reflector-bot[bot]" <130185838+oxide-reflector-bot[bot]@users.noreply.github.com> Date: Thu, 16 Nov 2023 17:02:11 +0000 Subject: [PATCH 06/11] Update maghemite to `12b392b` (#4505) --- package-manifest.toml | 12 ++++++------ tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 4 ++-- tools/maghemite_mgd_checksums | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package-manifest.toml b/package-manifest.toml index c65c5b6933..f320215a13 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -425,10 +425,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" +source.commit = "12b392be94ff93abc3017bf2610a3b18e2174a2d" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt -source.sha256 = "d871406ed926571efebdab248de08d4f1ca6c31d4f9a691ce47b186474165c57" +source.sha256 = "38851c79c85d53e997db748520fb27c82299ce7e58a550e35646a548498f1271" output.type = "tarball" [package.mg-ddm] @@ -441,10 +441,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" +source.commit = "12b392be94ff93abc3017bf2610a3b18e2174a2d" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "85ec05a8726989b5cb0a567de6b0855f6f84b6f3409ac99ccaf372be5821e45d" +source.sha256 = "8cd94e9a6f6175081ce78f0281085a08a5306cde453d8e21deb28050945b1d88" output.type = "zone" output.intermediate_only = true @@ -456,10 +456,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" +source.commit = "12b392be94ff93abc3017bf2610a3b18e2174a2d" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "aa7241cd35976f28f25aaf3ce2ce2af14dae1da9d67585c7de3b724dbcc55e60" +source.sha256 = "c4a7a626c84a28de3d2c6bfd85592bda2abad8cf5b41b2ce90b9c03904ccd3df" output.type = "zone" output.intermediate_only = true diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 40db886f69..76bdb9ca92 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" +COMMIT="12b392be94ff93abc3017bf2610a3b18e2174a2d" SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index ad88fef13e..d6d1788cbc 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" -SHA2="b3f55fe24e54530fdf96c22a033f9edc0bad9c0a5e3344763a23e52b251d5113" +COMMIT="12b392be94ff93abc3017bf2610a3b18e2174a2d" +SHA2="6c1fab8d5028b52a161d8bf02aae47844699cdc5f7b28e1ac519fc4ec1ab3971" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 7c1644b031..9657147159 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="aa7241cd35976f28f25aaf3ce2ce2af14dae1da9d67585c7de3b724dbcc55e60" -MGD_LINUX_SHA256="a39387c361ff2c2d0701d66c00b10e43c72fb5ddd1a5900b59ecccb832c80731" \ No newline at end of file +CIDL_SHA256="c4a7a626c84a28de3d2c6bfd85592bda2abad8cf5b41b2ce90b9c03904ccd3df" +MGD_LINUX_SHA256="81231b30872fa1c581aa22c101f32d11f33f335758ac1fd2653436fbc7aab93f" \ No newline at end of file From 2585b88d3b74976792d188fea6e79f4badd20077 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Fri, 17 Nov 2023 09:06:58 -0800 Subject: [PATCH 07/11] [db-queries] Avoid interactive transaction for all authz checks (#4506) While working on https://github.com/oxidecomputer/customer-support/issues/46 , I noticed that we perform an interactive transaction for every authz check perfomed by the database. This PR re-writes this code to avoid using an interactive transaction altogether. --- nexus/db-queries/src/db/datastore/role.rs | 69 +++++++++++------------ 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/role.rs b/nexus/db-queries/src/db/datastore/role.rs index b2ad441475..3a57ffc44c 100644 --- a/nexus/db-queries/src/db/datastore/role.rs +++ b/nexus/db-queries/src/db/datastore/role.rs @@ -127,7 +127,8 @@ impl DataStore { resource_type: ResourceType, resource_id: Uuid, ) -> Result, Error> { - use db::schema::role_assignment::dsl; + use db::schema::role_assignment::dsl as role_dsl; + use db::schema::silo_group_membership::dsl as group_dsl; // There is no resource-specific authorization check because all // authenticated users need to be able to list their own roles -- @@ -140,41 +141,39 @@ impl DataStore { // into some hurt by assigning loads of roles to someone and having that // person attempt to access anything. - self.pool_connection_authorized(opctx).await? - .transaction_async(|conn| async move { - let mut role_assignments = dsl::role_assignment - .filter(dsl::identity_type.eq(identity_type.clone())) - .filter(dsl::identity_id.eq(identity_id)) - .filter(dsl::resource_type.eq(resource_type.to_string())) - .filter(dsl::resource_id.eq(resource_id)) - .select(RoleAssignment::as_select()) - .load_async::(&conn) - .await?; - - // Return the roles that a silo user has from their group memberships - if identity_type == IdentityType::SiloUser { - use db::schema::silo_group_membership; - - let mut group_role_assignments = dsl::role_assignment - .filter(dsl::identity_type.eq(IdentityType::SiloGroup)) - .filter(dsl::identity_id.eq_any( - silo_group_membership::dsl::silo_group_membership - .filter(silo_group_membership::dsl::silo_user_id.eq(identity_id)) - .select(silo_group_membership::dsl::silo_group_id) - )) - .filter(dsl::resource_type.eq(resource_type.to_string())) - .filter(dsl::resource_id.eq(resource_id)) - .select(RoleAssignment::as_select()) - .load_async::(&conn) - .await?; - - role_assignments.append(&mut group_role_assignments); - } + let direct_roles_query = role_dsl::role_assignment + .filter(role_dsl::identity_type.eq(identity_type.clone())) + .filter(role_dsl::identity_id.eq(identity_id)) + .filter(role_dsl::resource_type.eq(resource_type.to_string())) + .filter(role_dsl::resource_id.eq(resource_id)) + .select(RoleAssignment::as_select()); + + let roles_from_groups_query = role_dsl::role_assignment + .filter(role_dsl::identity_type.eq(IdentityType::SiloGroup)) + .filter( + role_dsl::identity_id.eq_any( + group_dsl::silo_group_membership + .filter(group_dsl::silo_user_id.eq(identity_id)) + .select(group_dsl::silo_group_id), + ), + ) + .filter(role_dsl::resource_type.eq(resource_type.to_string())) + .filter(role_dsl::resource_id.eq(resource_id)) + .select(RoleAssignment::as_select()); - Ok(role_assignments) - }) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + let conn = self.pool_connection_authorized(opctx).await?; + if identity_type == IdentityType::SiloUser { + direct_roles_query + .union(roles_from_groups_query) + .load_async::(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } else { + direct_roles_query + .load_async::(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } } /// Fetches all of the externally-visible role assignments for the specified From 781ed1236489bb536506fb03ba1be39b34ed81a5 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 17 Nov 2023 17:30:02 +0000 Subject: [PATCH 08/11] Update `how-to-run.adoc` post-BGP (#4404) Small edits needed to bring this in line with how routes are configured now. `gateway_ip` is no longer explicitly named as such, so I've added an explicit comment to signpost how $GATEWAY_IP should be used w.r.t. the default route. --- docs/how-to-run.adoc | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 04d274da8b..f6d780ad72 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -266,26 +266,52 @@ last = "192.168.1.29" This is a range of IP addresses on your external network that Omicron can assign to externally-facing services (like DNS and the API). You'll need to change these if you've picked different addresses for your external network. See <<_external_networking>> above for more on this. +You will also need to update route information if your `$GATEWAY_IP` differs from the default. +The below example demonstrates a single static gateway route; in-depth explanations for testing with BGP can be found https://docs.oxide.computer/guides/system/network-preparations#_rack_switch_configuration_with_bgp[in the Network Preparations guide] and https://docs.oxide.computer/guides/operator/configuring-bgp[the Configuring BGP guide]: + [source,toml] ---- # Configuration to bring up boundary services and make Nexus reachable from the # outside. This block assumes that you're following option (2) above: putting # your Oxide system on an existing network that you control. [rack_network_config] -# The gateway for the external network -gateway_ip = "192.168.1.199" +# An internal-only IPv6 address block which contains AZ-wide services. +# This does not need to be changed. +rack_subnet = "fd00:1122:3344:01::/56" # A range of IP addresses used by Boundary Services on the network. In a real # system, these would be addresses of the uplink ports on the Sidecar. With # softnpu, only one address is used. infra_ip_first = "192.168.1.30" infra_ip_last = "192.168.1.30" -# Name of the port. This should always be "qsfp0" when using softnpu. -uplink_port = "qsfp0" -uplink_port_speed = "40G" -uplink_port_fec="none" + +# Configurations for BGP routers to run on the scrimlets. +# This array can typically be safely left empty for home/local use, +# otherwise this is a list of { asn: u32, originate: [""] } +# structs which will be be inserted when Nexus is started by sled-agent. +# See the 'Network Preparations' guide linked above. +bgp = [] + +[[rack_network_config.ports]] +# Routes associated with this port. +# NOTE: The below `nexthop` should be set to $GATEWAY_IP for your configuration +routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] +# Addresses associated with this port. # For softnpu, an address within the "infra" block above that will be used for # the softnpu uplink port. You can just pick the first address in that pool. -uplink_ip = "192.168.1.30" +addresses = ["192.168.1.30/32"] +# Name of the uplink port. This should always be "qsfp0" when using softnpu. +port = "qsfp0" +# The speed of this port. +uplink_port_speed = "40G" +# The forward error correction mode for this port. +uplink_port_fec="none" +# Switch to use for the uplink. For single-rack deployments this can be +# "switch0" (upper slot) or "switch1" (lower slot). For single-node softnpu +# and dendrite stub environments, use "switch0" +switch = "switch0" +# Neighbors we expect to peer with over BGP on this port. +# see: common/src/api/internal/shared.rs – BgpPeerConfig +bgp_peers = [] ---- In some configurations (not the one described here), it may be necessary to update `smf/sled-agent/$MACHINE/config.toml`: From 1860621199e1af8fd1e47a42a898d8ea9def5737 Mon Sep 17 00:00:00 2001 From: Rain Date: Fri, 17 Nov 2023 15:48:28 -0800 Subject: [PATCH 09/11] [ci] switch GHA to using the PR tip by default (#4462) Fixes #4461 (see that issue for more). --- .github/workflows/check-opte-ver.yml | 2 ++ .github/workflows/check-workspace-deps.yml | 2 ++ .github/workflows/hakari.yml | 2 ++ .github/workflows/rust.yml | 8 ++++++++ .github/workflows/validate-openapi-spec.yml | 2 ++ 5 files changed, 16 insertions(+) diff --git a/.github/workflows/check-opte-ver.yml b/.github/workflows/check-opte-ver.yml index 9fc390277b..42ef1dda11 100644 --- a/.github/workflows/check-opte-ver.yml +++ b/.github/workflows/check-opte-ver.yml @@ -10,6 +10,8 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - name: Install jq run: sudo apt-get install -y jq - name: Install toml-cli diff --git a/.github/workflows/check-workspace-deps.yml b/.github/workflows/check-workspace-deps.yml index 7ba0c66566..f94ed32fde 100644 --- a/.github/workflows/check-workspace-deps.yml +++ b/.github/workflows/check-workspace-deps.yml @@ -11,5 +11,7 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - name: Check Workspace Dependencies run: cargo xtask check-workspace-deps diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index 6f2dc04b91..07b7124f73 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -18,6 +18,8 @@ jobs: RUSTFLAGS: -D warnings steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + with: + ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 with: toolchain: stable diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f2581845d9..6239add88f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -10,6 +10,8 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - name: Report cargo version run: cargo --version - name: Report rustfmt version @@ -30,6 +32,8 @@ jobs: - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version @@ -58,6 +62,8 @@ jobs: - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version @@ -86,6 +92,8 @@ jobs: - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version diff --git a/.github/workflows/validate-openapi-spec.yml b/.github/workflows/validate-openapi-spec.yml index 2716c0571f..ea77ed9497 100644 --- a/.github/workflows/validate-openapi-spec.yml +++ b/.github/workflows/validate-openapi-spec.yml @@ -11,6 +11,8 @@ jobs: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2 with: node-version: '18' From d13a0dc8f8609daf57be6725bc1c5467ea7e1fe4 Mon Sep 17 00:00:00 2001 From: Rain Date: Fri, 17 Nov 2023 17:02:02 -0800 Subject: [PATCH 10/11] [wicket] produce a better error message if MGS isn't available (#4510) Currently, we just time out in this case. Produce a better error message by waiting for 80% of `WICKETD_TIMEOUT`. --- wicket-common/src/lib.rs | 7 ++++++ wicket/src/cli/rack_update.rs | 3 ++- wicket/src/wicketd.rs | 5 +---- wicketd/src/http_entrypoints.rs | 40 +++++++++++++++++++++++++-------- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/wicket-common/src/lib.rs b/wicket-common/src/lib.rs index 9e92d20c0a..aea0634ce7 100644 --- a/wicket-common/src/lib.rs +++ b/wicket-common/src/lib.rs @@ -4,6 +4,13 @@ // Copyright 2023 Oxide Computer Company +use std::time::Duration; + pub mod rack_setup; pub mod rack_update; pub mod update_events; + +// WICKETD_TIMEOUT used to be 1 second, but that might be too short (and in +// particular might be responsible for +// https://github.com/oxidecomputer/omicron/issues/3103). +pub const WICKETD_TIMEOUT: Duration = Duration::from_secs(5); diff --git a/wicket/src/cli/rack_update.rs b/wicket/src/cli/rack_update.rs index f539c22c35..fa41fa7b8c 100644 --- a/wicket/src/cli/rack_update.rs +++ b/wicket/src/cli/rack_update.rs @@ -22,6 +22,7 @@ use update_engine::{ }; use wicket_common::{ rack_update::ClearUpdateStateResponse, update_events::EventReport, + WICKETD_TIMEOUT, }; use wicketd_client::types::{ClearUpdateStateParams, StartUpdateParams}; @@ -31,7 +32,7 @@ use crate::{ parse_event_report_map, ComponentId, CreateClearUpdateStateOptions, CreateStartUpdateOptions, }, - wicketd::{create_wicketd_client, WICKETD_TIMEOUT}, + wicketd::create_wicketd_client, }; use super::command::CommandOutput; diff --git a/wicket/src/wicketd.rs b/wicket/src/wicketd.rs index ec1130a594..a951bf428b 100644 --- a/wicket/src/wicketd.rs +++ b/wicket/src/wicketd.rs @@ -10,6 +10,7 @@ use std::net::SocketAddrV6; use tokio::sync::mpsc::{self, Sender, UnboundedSender}; use tokio::time::{interval, Duration, MissedTickBehavior}; use wicket_common::rack_update::{SpIdentifier, SpType}; +use wicket_common::WICKETD_TIMEOUT; use wicketd_client::types::{ AbortUpdateOptions, ClearUpdateStateOptions, ClearUpdateStateParams, GetInventoryParams, GetInventoryResponse, GetLocationResponse, @@ -38,10 +39,6 @@ impl From for SpIdentifier { } const WICKETD_POLL_INTERVAL: Duration = Duration::from_millis(500); -// WICKETD_TIMEOUT used to be 1 second, but that might be too short (and in -// particular might be responsible for -// https://github.com/oxidecomputer/omicron/issues/3103). -pub(crate) const WICKETD_TIMEOUT: Duration = Duration::from_secs(5); // Assume that these requests are periodic on the order of seconds or the // result of human interaction. In either case, this buffer should be plenty diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs index d6cb6ebd6d..dbd3e31072 100644 --- a/wicketd/src/http_entrypoints.rs +++ b/wicketd/src/http_entrypoints.rs @@ -51,6 +51,7 @@ use std::time::Duration; use tokio::io::AsyncWriteExt; use wicket_common::rack_setup::PutRssUserConfigInsensitive; use wicket_common::update_events::EventReport; +use wicket_common::WICKETD_TIMEOUT; use crate::ServerContext; @@ -896,21 +897,42 @@ async fn post_start_update( // 1. We haven't pulled its state in our inventory (most likely cause: the // cubby is empty; less likely cause: the SP is misbehaving, which will // make updating it very unlikely to work anyway) - // 2. We have pulled its state but our hardware manager says we can't update - // it (most likely cause: the target is the sled we're currently running - // on; less likely cause: our hardware manager failed to get our local - // identifying information, and it refuses to update this target out of - // an abundance of caution). + // 2. We have pulled its state but our hardware manager says we can't + // update it (most likely cause: the target is the sled we're currently + // running on; less likely cause: our hardware manager failed to get our + // local identifying information, and it refuses to update this target + // out of an abundance of caution). // - // First, get our most-recently-cached inventory view. - let inventory = match rqctx.mgs_handle.get_cached_inventory().await { - Ok(inventory) => inventory, - Err(ShutdownInProgress) => { + // First, get our most-recently-cached inventory view. (Only wait 80% of + // WICKETD_TIMEOUT for this: if even a cached inventory isn't available, + // it's because we've never established contact with MGS. In that case, we + // should produce a useful error message rather than timing out on the + // client.) + let inventory = match tokio::time::timeout( + WICKETD_TIMEOUT.mul_f32(0.8), + rqctx.mgs_handle.get_cached_inventory(), + ) + .await + { + Ok(Ok(inventory)) => inventory, + Ok(Err(ShutdownInProgress)) => { return Err(HttpError::for_unavail( None, "Server is shutting down".into(), )); } + Err(_) => { + // Have to construct an HttpError manually because + // HttpError::for_unavail doesn't accept an external message. + let message = + "Rack inventory not yet available (is MGS alive?)".to_owned(); + return Err(HttpError { + status_code: http::StatusCode::SERVICE_UNAVAILABLE, + error_code: None, + external_message: message.clone(), + internal_message: message, + }); + } }; // Error cases. From 14f8f3159d40c7d044d12f1424d209cbc100a9cb Mon Sep 17 00:00:00 2001 From: Levon Tarver <11586085+internet-diglett@users.noreply.github.com> Date: Fri, 17 Nov 2023 22:22:17 -0600 Subject: [PATCH 11/11] NAT RPW (#3804) * Add db table for tracking nat entries * Add endpoint for retrieving changesets * Update instance sagas to update table and trigger RPW * Periodically cleanup soft-deleted entries that no longer need to be sync'd by dendrite. The other half of the RPW lives in Dendrite. It will periodically check for a changeset, or check for a changeset when the trigger endpoint is called by the relevant saga / nexus operation. --- common/src/api/external/mod.rs | 1 + common/src/nexus_config.rs | 17 +- dev-tools/omdb/src/bin/omdb/nexus.rs | 1 + dev-tools/omdb/tests/env.out | 15 + dev-tools/omdb/tests/successes.out | 12 + nexus/db-model/src/ipv4_nat_entry.rs | 81 ++++ nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/schema.rs | 28 +- .../src/db/datastore/ipv4_nat_entry.rs | 440 ++++++++++++++++++ nexus/db-queries/src/db/datastore/mod.rs | 1 + nexus/examples/config.toml | 1 + nexus/src/app/background/init.rs | 24 + nexus/src/app/background/mod.rs | 1 + nexus/src/app/background/nat_cleanup.rs | 111 +++++ nexus/src/app/instance_network.rs | 239 ++++++---- nexus/src/app/mod.rs | 1 + nexus/src/internal_api/http_entrypoints.rs | 51 ++ nexus/tests/config.test.toml | 1 + openapi/nexus-internal.json | 104 +++++ package-manifest.toml | 12 +- schema/crdb/{10.0.0 => 11.0.0}/README.md | 0 schema/crdb/{10.0.0 => 11.0.0}/up01.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up02.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up03.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up04.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up05.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up06.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up07.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up08.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up09.sql | 0 schema/crdb/11.0.0/up1.sql | 1 + schema/crdb/{10.0.0 => 11.0.0}/up10.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up11.sql | 0 schema/crdb/{10.0.0 => 11.0.0}/up12.sql | 0 schema/crdb/11.0.0/up2.sql | 13 + schema/crdb/11.0.0/up3.sql | 13 + schema/crdb/11.0.0/up4.sql | 5 + schema/crdb/11.0.0/up5.sql | 1 + schema/crdb/11.0.0/up6.sql | 13 + schema/crdb/dbinit.sql | 80 +++- smf/nexus/multi-sled/config-partial.toml | 1 + smf/nexus/single-sled/config-partial.toml | 1 + tools/dendrite_openapi_version | 4 +- tools/dendrite_stub_checksums | 6 +- 44 files changed, 1168 insertions(+), 113 deletions(-) create mode 100644 nexus/db-model/src/ipv4_nat_entry.rs create mode 100644 nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs create mode 100644 nexus/src/app/background/nat_cleanup.rs rename schema/crdb/{10.0.0 => 11.0.0}/README.md (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up01.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up02.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up03.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up04.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up05.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up06.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up07.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up08.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up09.sql (100%) create mode 100644 schema/crdb/11.0.0/up1.sql rename schema/crdb/{10.0.0 => 11.0.0}/up10.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up11.sql (100%) rename schema/crdb/{10.0.0 => 11.0.0}/up12.sql (100%) create mode 100644 schema/crdb/11.0.0/up2.sql create mode 100644 schema/crdb/11.0.0/up3.sql create mode 100644 schema/crdb/11.0.0/up4.sql create mode 100644 schema/crdb/11.0.0/up5.sql create mode 100644 schema/crdb/11.0.0/up6.sql diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index fcea57220d..adf661516a 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -750,6 +750,7 @@ pub enum ResourceType { UserBuiltin, Zpool, Vmm, + Ipv4NatEntry, } // IDENTITY METADATA diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index 4e821e2676..94c39b4436 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -335,6 +335,8 @@ pub struct BackgroundTaskConfig { pub dns_external: DnsTasksConfig, /// configuration for external endpoint list watcher pub external_endpoints: ExternalEndpointsConfig, + /// configuration for nat table garbage collector + pub nat_cleanup: NatCleanupConfig, /// configuration for inventory tasks pub inventory: InventoryConfig, } @@ -371,6 +373,14 @@ pub struct ExternalEndpointsConfig { // allow/disallow wildcard certs, don't serve expired certs, etc.) } +#[serde_as] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct NatCleanupConfig { + /// period (in seconds) for periodic activations of this background task + #[serde_as(as = "DurationSeconds")] + pub period_secs: Duration, +} + #[serde_as] #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct InventoryConfig { @@ -498,7 +508,7 @@ mod test { BackgroundTaskConfig, Config, ConfigDropshotWithTls, ConsoleConfig, Database, DeploymentConfig, DnsTasksConfig, DpdConfig, ExternalEndpointsConfig, InternalDns, InventoryConfig, LoadError, - LoadErrorKind, MgdConfig, PackageConfig, SchemeName, + LoadErrorKind, MgdConfig, NatCleanupConfig, PackageConfig, SchemeName, TimeseriesDbConfig, Tunables, UpdatesConfig, }; use crate::address::{Ipv6Subnet, RACK_PREFIX}; @@ -649,6 +659,7 @@ mod test { dns_external.period_secs_propagation = 7 dns_external.max_concurrent_server_updates = 8 external_endpoints.period_secs = 9 + nat_cleanup.period_secs = 30 inventory.period_secs = 10 inventory.nkeep = 11 inventory.disable = false @@ -746,6 +757,9 @@ mod test { external_endpoints: ExternalEndpointsConfig { period_secs: Duration::from_secs(9), }, + nat_cleanup: NatCleanupConfig { + period_secs: Duration::from_secs(30), + }, inventory: InventoryConfig { period_secs: Duration::from_secs(10), nkeep: 11, @@ -804,6 +818,7 @@ mod test { dns_external.period_secs_propagation = 7 dns_external.max_concurrent_server_updates = 8 external_endpoints.period_secs = 9 + nat_cleanup.period_secs = 30 inventory.period_secs = 10 inventory.nkeep = 3 inventory.disable = false diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index 128d4315f2..9f91d38504 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -159,6 +159,7 @@ async fn cmd_nexus_background_tasks_show( "dns_config_external", "dns_servers_external", "dns_propagation_external", + "nat_v4_garbage_collector", ] { if let Some(bgtask) = tasks.remove(name) { print_task(&bgtask); diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 7949c1eb61..fd50d80c81 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -61,6 +61,11 @@ task: "inventory_collection" collects hardware and software inventory data from the whole system +task: "nat_v4_garbage_collector" + prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a + predetermined retention policy + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT @@ -121,6 +126,11 @@ task: "inventory_collection" collects hardware and software inventory data from the whole system +task: "nat_v4_garbage_collector" + prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a + predetermined retention policy + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. @@ -168,6 +178,11 @@ task: "inventory_collection" collects hardware and software inventory data from the whole system +task: "nat_v4_garbage_collector" + prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a + predetermined retention policy + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 8162b6d9de..6bc3a85e8a 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -255,6 +255,11 @@ task: "inventory_collection" collects hardware and software inventory data from the whole system +task: "nat_v4_garbage_collector" + prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a + predetermined retention policy + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ @@ -319,6 +324,13 @@ task: "dns_propagation_external" [::1]:REDACTED_PORT success +task: "nat_v4_garbage_collector" + configured period: every 30s + currently executing: no + last completed activation: iter 2, triggered by an explicit signal + started at (s ago) and ran for ms +warning: unknown background task: "nat_v4_garbage_collector" (don't know how to interpret details: Null) + task: "external_endpoints" configured period: every 1m currently executing: no diff --git a/nexus/db-model/src/ipv4_nat_entry.rs b/nexus/db-model/src/ipv4_nat_entry.rs new file mode 100644 index 0000000000..570a46b5e9 --- /dev/null +++ b/nexus/db-model/src/ipv4_nat_entry.rs @@ -0,0 +1,81 @@ +use std::net::{Ipv4Addr, Ipv6Addr}; + +use super::MacAddr; +use crate::{schema::ipv4_nat_entry, Ipv4Net, Ipv6Net, SqlU16, Vni}; +use chrono::{DateTime, Utc}; +use omicron_common::api::external; +use schemars::JsonSchema; +use serde::Serialize; +use uuid::Uuid; + +/// Values used to create an Ipv4NatEntry +#[derive(Insertable, Debug, Clone)] +#[diesel(table_name = ipv4_nat_entry)] +pub struct Ipv4NatValues { + pub external_address: Ipv4Net, + pub first_port: SqlU16, + pub last_port: SqlU16, + pub sled_address: Ipv6Net, + pub vni: Vni, + pub mac: MacAddr, +} + +/// Database representation of an Ipv4 NAT Entry. +#[derive(Queryable, Debug, Clone, Selectable)] +#[diesel(table_name = ipv4_nat_entry)] +pub struct Ipv4NatEntry { + pub id: Uuid, + pub external_address: Ipv4Net, + pub first_port: SqlU16, + pub last_port: SqlU16, + pub sled_address: Ipv6Net, + pub vni: Vni, + pub mac: MacAddr, + pub version_added: i64, + pub version_removed: Option, + pub time_created: DateTime, + pub time_deleted: Option>, +} + +impl Ipv4NatEntry { + pub fn first_port(&self) -> u16 { + self.first_port.into() + } + + pub fn last_port(&self) -> u16 { + self.last_port.into() + } +} + +/// NAT Record +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct Ipv4NatEntryView { + pub external_address: Ipv4Addr, + pub first_port: u16, + pub last_port: u16, + pub sled_address: Ipv6Addr, + pub vni: external::Vni, + pub mac: external::MacAddr, + pub gen: i64, + pub deleted: bool, +} + +impl From for Ipv4NatEntryView { + fn from(value: Ipv4NatEntry) -> Self { + let (gen, deleted) = match value.version_removed { + Some(gen) => (gen, true), + None => (value.version_added, false), + }; + + Self { + external_address: value.external_address.ip(), + first_port: value.first_port(), + last_port: value.last_port(), + sled_address: value.sled_address.ip(), + vni: value.vni.0, + mac: *value.mac, + gen, + deleted, + } + } +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 7aa8a6b076..6b65eb87ec 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -53,6 +53,7 @@ mod system_update; // These actually represent subqueries, not real table. // However, they must be defined in the same crate as our tables // for join-based marker trait generation. +mod ipv4_nat_entry; pub mod queries; mod rack; mod region; @@ -124,6 +125,7 @@ pub use instance_cpu_count::*; pub use instance_state::*; pub use inventory::*; pub use ip_pool::*; +pub use ipv4_nat_entry::*; pub use ipv4net::*; pub use ipv6::*; pub use ipv6net::*; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 7c6b8bbd0a..4844f2a33f 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -489,6 +489,32 @@ table! { } } +table! { + ipv4_nat_entry (id) { + id -> Uuid, + external_address -> Inet, + first_port -> Int4, + last_port -> Int4, + sled_address -> Inet, + vni -> Int4, + mac -> Int8, + version_added -> Int8, + version_removed -> Nullable, + time_created -> Timestamptz, + time_deleted -> Nullable, + } +} + +// This is the sequence used for the version number +// in ipv4_nat_entry. +table! { + ipv4_nat_version (last_value) { + last_value -> Int8, + log_cnt -> Int8, + is_called -> Bool, + } +} + table! { external_ip (id) { id -> Uuid, @@ -1243,7 +1269,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(10, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(11, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs new file mode 100644 index 0000000000..274937b299 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs @@ -0,0 +1,440 @@ +use super::DataStore; +use crate::context::OpContext; +use crate::db; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::model::{Ipv4NatEntry, Ipv4NatValues}; +use async_bb8_diesel::AsyncRunQueryDsl; +use chrono::{DateTime, Utc}; +use diesel::prelude::*; +use diesel::sql_types::BigInt; +use nexus_db_model::ExternalIp; +use nexus_db_model::Ipv4NatEntryView; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::LookupResult; +use omicron_common::api::external::LookupType; +use omicron_common::api::external::ResourceType; + +impl DataStore { + pub async fn ensure_ipv4_nat_entry( + &self, + opctx: &OpContext, + nat_entry: Ipv4NatValues, + ) -> CreateResult<()> { + use db::schema::ipv4_nat_entry::dsl; + use diesel::sql_types; + + // Look up any NAT entries that already have the exact parameters + // we're trying to INSERT. + let matching_entry_subquery = dsl::ipv4_nat_entry + .filter(dsl::external_address.eq(nat_entry.external_address)) + .filter(dsl::first_port.eq(nat_entry.first_port)) + .filter(dsl::last_port.eq(nat_entry.last_port)) + .filter(dsl::sled_address.eq(nat_entry.sled_address)) + .filter(dsl::vni.eq(nat_entry.vni)) + .filter(dsl::mac.eq(nat_entry.mac)) + .select(( + dsl::external_address, + dsl::first_port, + dsl::last_port, + dsl::sled_address, + dsl::vni, + dsl::mac, + )); + + // SELECT exactly the values we're trying to INSERT, but only + // if it does not already exist. + let new_entry_subquery = diesel::dsl::select(( + nat_entry.external_address.into_sql::(), + nat_entry.first_port.into_sql::(), + nat_entry.last_port.into_sql::(), + nat_entry.sled_address.into_sql::(), + nat_entry.vni.into_sql::(), + nat_entry.mac.into_sql::(), + )) + .filter(diesel::dsl::not(diesel::dsl::exists(matching_entry_subquery))); + + diesel::insert_into(dsl::ipv4_nat_entry) + .values(new_entry_subquery) + .into_columns(( + dsl::external_address, + dsl::first_port, + dsl::last_port, + dsl::sled_address, + dsl::vni, + dsl::mac, + )) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } + + pub async fn ipv4_nat_delete( + &self, + opctx: &OpContext, + nat_entry: &Ipv4NatEntry, + ) -> DeleteResult { + use db::schema::ipv4_nat_entry::dsl; + + let updated_rows = diesel::update(dsl::ipv4_nat_entry) + .set(( + dsl::version_removed.eq(ipv4_nat_next_version().nullable()), + dsl::time_deleted.eq(Utc::now()), + )) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::version_removed.is_null()) + .filter(dsl::id.eq(nat_entry.id)) + .filter(dsl::version_added.eq(nat_entry.version_added)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + if updated_rows == 0 { + return Err(Error::ObjectNotFound { + type_name: ResourceType::Ipv4NatEntry, + lookup_type: LookupType::ByCompositeId( + "id, version_added".to_string(), + ), + }); + } + Ok(()) + } + + pub async fn ipv4_nat_find_by_id( + &self, + opctx: &OpContext, + id: uuid::Uuid, + ) -> LookupResult { + use db::schema::ipv4_nat_entry::dsl; + + let result = dsl::ipv4_nat_entry + .filter(dsl::id.eq(id)) + .select(Ipv4NatEntry::as_select()) + .limit(1) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + if let Some(nat_entry) = result.first() { + Ok(nat_entry.clone()) + } else { + Err(Error::InvalidRequest { + message: "no matching records".to_string(), + }) + } + } + + pub async fn ipv4_nat_delete_by_external_ip( + &self, + opctx: &OpContext, + external_ip: &ExternalIp, + ) -> DeleteResult { + use db::schema::ipv4_nat_entry::dsl; + + let updated_rows = diesel::update(dsl::ipv4_nat_entry) + .set(( + dsl::version_removed.eq(ipv4_nat_next_version().nullable()), + dsl::time_deleted.eq(Utc::now()), + )) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::version_removed.is_null()) + .filter(dsl::external_address.eq(external_ip.ip)) + .filter(dsl::first_port.eq(external_ip.first_port)) + .filter(dsl::last_port.eq(external_ip.last_port)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + if updated_rows == 0 { + return Err(Error::ObjectNotFound { + type_name: ResourceType::Ipv4NatEntry, + lookup_type: LookupType::ByCompositeId( + "external_ip, first_port, last_port".to_string(), + ), + }); + } + Ok(()) + } + + pub async fn ipv4_nat_find_by_values( + &self, + opctx: &OpContext, + values: Ipv4NatValues, + ) -> LookupResult { + use db::schema::ipv4_nat_entry::dsl; + let result = dsl::ipv4_nat_entry + .filter(dsl::external_address.eq(values.external_address)) + .filter(dsl::first_port.eq(values.first_port)) + .filter(dsl::last_port.eq(values.last_port)) + .filter(dsl::mac.eq(values.mac)) + .filter(dsl::sled_address.eq(values.sled_address)) + .filter(dsl::vni.eq(values.vni)) + .filter(dsl::time_deleted.is_null()) + .select(Ipv4NatEntry::as_select()) + .limit(1) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + if let Some(nat_entry) = result.first() { + Ok(nat_entry.clone()) + } else { + Err(Error::InvalidRequest { + message: "no matching records".to_string(), + }) + } + } + + pub async fn ipv4_nat_list_since_version( + &self, + opctx: &OpContext, + version: i64, + limit: u32, + ) -> ListResultVec { + use db::schema::ipv4_nat_entry::dsl; + + let list = dsl::ipv4_nat_entry + .filter( + dsl::version_added + .gt(version) + .or(dsl::version_removed.gt(version)), + ) + .limit(limit as i64) + .select(Ipv4NatEntry::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(list) + } + + pub async fn ipv4_nat_changeset( + &self, + opctx: &OpContext, + version: i64, + limit: u32, + ) -> ListResultVec { + let nat_entries = + self.ipv4_nat_list_since_version(opctx, version, limit).await?; + let nat_entries: Vec = + nat_entries.iter().map(|e| e.clone().into()).collect(); + Ok(nat_entries) + } + + pub async fn ipv4_nat_current_version( + &self, + opctx: &OpContext, + ) -> LookupResult { + use db::schema::ipv4_nat_version::dsl; + + let latest: Option = dsl::ipv4_nat_version + .select(diesel::dsl::max(dsl::last_value)) + .first_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + match latest { + Some(value) => Ok(value), + None => Err(Error::InvalidRequest { + message: "sequence table is empty!".to_string(), + }), + } + } + + pub async fn ipv4_nat_cleanup( + &self, + opctx: &OpContext, + version: i64, + before_timestamp: DateTime, + ) -> DeleteResult { + use db::schema::ipv4_nat_entry::dsl; + + diesel::delete(dsl::ipv4_nat_entry) + .filter(dsl::version_removed.lt(version)) + .filter(dsl::time_deleted.lt(before_timestamp)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } +} + +fn ipv4_nat_next_version() -> diesel::expression::SqlLiteral { + diesel::dsl::sql::("nextval('omicron.public.ipv4_nat_version')") +} + +#[cfg(test)] +mod test { + use std::str::FromStr; + + use crate::db::datastore::datastore_test; + use chrono::Utc; + use nexus_db_model::{Ipv4NatValues, MacAddr, Vni}; + use nexus_test_utils::db::test_setup_database; + use omicron_common::api::external; + use omicron_test_utils::dev; + + // Test our ability to track additions and deletions since a given version number + #[tokio::test] + async fn nat_version_tracking() { + let logctx = dev::test_setup_log("test_nat_version_tracking"); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // We should not have any NAT entries at this moment + let initial_state = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert!(initial_state.is_empty()); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 0 + ); + + // Each change (creation / deletion) to the NAT table should increment the + // version number of the row in the NAT table + let external_address = external::Ipv4Net( + ipnetwork::Ipv4Network::try_from("10.0.0.100").unwrap(), + ); + + let sled_address = external::Ipv6Net( + ipnetwork::Ipv6Network::try_from("fd00:1122:3344:104::1").unwrap(), + ); + + // Add a nat entry. + let nat1 = Ipv4NatValues { + external_address: external_address.into(), + first_port: 0.into(), + last_port: 999.into(), + sled_address: sled_address.into(), + vni: Vni(external::Vni::random()), + mac: MacAddr( + external::MacAddr::from_str("A8:40:25:F5:EB:2A").unwrap(), + ), + }; + + datastore.ensure_ipv4_nat_entry(&opctx, nat1.clone()).await.unwrap(); + let first_entry = + datastore.ipv4_nat_find_by_values(&opctx, nat1).await.unwrap(); + + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + // The NAT table has undergone one change. One entry has been added, + // none deleted, so we should be at version 1. + assert_eq!(nat_entries.len(), 1); + assert_eq!(nat_entries.last().unwrap().version_added, 1); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 1 + ); + + // Add another nat entry. + let nat2 = Ipv4NatValues { + external_address: external_address.into(), + first_port: 1000.into(), + last_port: 1999.into(), + sled_address: sled_address.into(), + vni: Vni(external::Vni::random()), + mac: MacAddr( + external::MacAddr::from_str("A8:40:25:F5:EB:2B").unwrap(), + ), + }; + + datastore.ensure_ipv4_nat_entry(&opctx, nat2).await.unwrap(); + + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + // The NAT table has undergone two changes. Two entries have been + // added, none deleted, so we should be at version 2. + let nat_entry = + nat_entries.iter().find(|e| e.version_added == 2).unwrap(); + assert_eq!(nat_entries.len(), 2); + assert_eq!(nat_entry.version_added, 2); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 2 + ); + + // Test Cleanup logic + // Cleanup should only perma-delete entries that are older than a + // specified version number and whose `time_deleted` field is + // older than a specified age. + let time_cutoff = Utc::now(); + datastore.ipv4_nat_cleanup(&opctx, 2, time_cutoff).await.unwrap(); + + // Nothing should have changed (no records currently marked for deletion) + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert_eq!(nat_entries.len(), 2); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 2 + ); + + // Delete the first nat entry. It should show up as a later version number. + datastore.ipv4_nat_delete(&opctx, &first_entry).await.unwrap(); + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + // The NAT table has undergone three changes. Two entries have been + // added, one deleted, so we should be at version 3. Since the + // first entry was marked for deletion (and it was the third change), + // the first entry's version number should now be 3. + let nat_entry = + nat_entries.iter().find(|e| e.version_removed.is_some()).unwrap(); + assert_eq!(nat_entries.len(), 2); + assert_eq!(nat_entry.version_removed, Some(3)); + assert_eq!(nat_entry.id, first_entry.id); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 3 + ); + + // Try cleaning up with the old version and time cutoff values + datastore.ipv4_nat_cleanup(&opctx, 2, time_cutoff).await.unwrap(); + + // Try cleaning up with a greater version and old time cutoff values + datastore.ipv4_nat_cleanup(&opctx, 6, time_cutoff).await.unwrap(); + + // Try cleaning up with a older version and newer time cutoff values + datastore.ipv4_nat_cleanup(&opctx, 2, Utc::now()).await.unwrap(); + + // Both records should still exist (soft deleted record is newer than cutoff + // values ) + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert_eq!(nat_entries.len(), 2); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 3 + ); + + // Try cleaning up with a both cutoff values increased + datastore.ipv4_nat_cleanup(&opctx, 4, Utc::now()).await.unwrap(); + + // Soft deleted NAT entry should be removed from the table + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert_eq!(nat_entries.len(), 1); + + // version should be unchanged + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 3 + ); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 91373f6875..7385970fb1 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -63,6 +63,7 @@ mod image; mod instance; mod inventory; mod ip_pool; +mod ipv4_nat_entry; mod network_interface; mod oximeter; mod physical_disk; diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index efc9aa9c27..3679fa8196 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -92,6 +92,7 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 +nat_cleanup.period_secs = 30 # How frequently to collect hardware/software inventory from the whole system # (even if we don't have reason to believe anything has changed). inventory.period_secs = 600 diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index b000dd9bda..d27248ffdc 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -10,12 +10,15 @@ use super::dns_propagation; use super::dns_servers; use super::external_endpoints; use super::inventory_collection; +use super::nat_cleanup; use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; +use omicron_common::api::internal::shared::SwitchLocation; use omicron_common::nexus_config::BackgroundTaskConfig; use omicron_common::nexus_config::DnsTasksConfig; use std::collections::BTreeMap; +use std::collections::HashMap; use std::sync::Arc; use uuid::Uuid; @@ -44,6 +47,8 @@ pub struct BackgroundTasks { pub external_endpoints: tokio::sync::watch::Receiver< Option, >, + /// task handle for the ipv4 nat entry garbage collector + pub nat_cleanup: common::TaskHandle, /// task handle for the task that collects inventory pub task_inventory_collection: common::TaskHandle, @@ -55,6 +60,7 @@ impl BackgroundTasks { opctx: &OpContext, datastore: Arc, config: &BackgroundTaskConfig, + dpd_clients: &HashMap>, nexus_id: Uuid, resolver: internal_dns::resolver::Resolver, ) -> BackgroundTasks { @@ -96,6 +102,23 @@ impl BackgroundTasks { (task, watcher_channel) }; + let nat_cleanup = { + driver.register( + "nat_v4_garbage_collector".to_string(), + String::from( + "prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table \ + based on a predetermined retention policy", + ), + config.nat_cleanup.period_secs, + Box::new(nat_cleanup::Ipv4NatGarbageCollector::new( + datastore.clone(), + dpd_clients.values().map(|client| client.clone()).collect(), + )), + opctx.child(BTreeMap::new()), + vec![], + ) + }; + // Background task: inventory collector let task_inventory_collection = { let collector = inventory_collection::InventoryCollector::new( @@ -128,6 +151,7 @@ impl BackgroundTasks { task_external_dns_servers, task_external_endpoints, external_endpoints, + nat_cleanup, task_inventory_collection, } } diff --git a/nexus/src/app/background/mod.rs b/nexus/src/app/background/mod.rs index e1f474b41a..954207cb3c 100644 --- a/nexus/src/app/background/mod.rs +++ b/nexus/src/app/background/mod.rs @@ -11,6 +11,7 @@ mod dns_servers; mod external_endpoints; mod init; mod inventory_collection; +mod nat_cleanup; mod status; pub use common::Driver; diff --git a/nexus/src/app/background/nat_cleanup.rs b/nexus/src/app/background/nat_cleanup.rs new file mode 100644 index 0000000000..1691d96a4b --- /dev/null +++ b/nexus/src/app/background/nat_cleanup.rs @@ -0,0 +1,111 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Background task for garbage collecting ipv4_nat_entry table. +//! Responsible for cleaning up soft deleted entries once they +//! have been propagated to running dpd instances. + +use super::common::BackgroundTask; +use chrono::{Duration, Utc}; +use futures::future::BoxFuture; +use futures::FutureExt; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use serde_json::json; +use std::sync::Arc; + +/// Background task that periodically prunes soft-deleted entries +/// from ipv4_nat_entry table +pub struct Ipv4NatGarbageCollector { + datastore: Arc, + dpd_clients: Vec>, +} + +impl Ipv4NatGarbageCollector { + pub fn new( + datastore: Arc, + dpd_clients: Vec>, + ) -> Ipv4NatGarbageCollector { + Ipv4NatGarbageCollector { datastore, dpd_clients } + } +} + +impl BackgroundTask for Ipv4NatGarbageCollector { + fn activate<'a, 'b, 'c>( + &'a mut self, + opctx: &'b OpContext, + ) -> BoxFuture<'c, serde_json::Value> + where + 'a: 'c, + 'b: 'c, + { + async { + let log = &opctx.log; + + let result = self.datastore.ipv4_nat_current_version(opctx).await; + + let mut min_gen = match result { + Ok(gen) => gen, + Err(error) => { + warn!( + &log, + "failed to read generation of database"; + "error" => format!("{:#}", error) + ); + return json!({ + "error": + format!( + "failed to read generation of database: \ + {:#}", + error + ) + }); + } + }; + + for client in &self.dpd_clients { + let response = client.ipv4_nat_generation().await; + match response { + Ok(gen) => min_gen = std::cmp::min(min_gen, *gen), + Err(error) => { + warn!( + &log, + "failed to read generation of dpd"; + "error" => format!("{:#}", error) + ); + return json!({ + "error": + format!( + "failed to read generation of dpd: \ + {:#}", + error + ) + }); + } + } + } + + let retention_threshold = Utc::now() - Duration::weeks(2); + + let result = self + .datastore + .ipv4_nat_cleanup(opctx, min_gen, retention_threshold) + .await + .unwrap(); + + let rv = serde_json::to_value(&result).unwrap_or_else(|error| { + json!({ + "error": + format!( + "failed to serialize final value: {:#}", + error + ) + }) + }); + + rv + } + .boxed() + } +} diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs index 0f52cbd260..abb8c744e1 100644 --- a/nexus/src/app/instance_network.rs +++ b/nexus/src/app/instance_network.rs @@ -5,6 +5,10 @@ //! Routines that manage instance-related networking state. use crate::app::sagas::retry_until_known_result; +use ipnetwork::IpNetwork; +use ipnetwork::Ipv6Network; +use nexus_db_model::Ipv4NatValues; +use nexus_db_model::Vni as DbVni; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -12,6 +16,8 @@ use nexus_db_queries::db::identity::Asset; use nexus_db_queries::db::lookup::LookupPath; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; +use omicron_common::api::external::Ipv4Net; +use omicron_common::api::external::Ipv6Net; use omicron_common::api::internal::nexus; use omicron_common::api::internal::shared::SwitchLocation; use sled_agent_client::types::DeleteVirtualNetworkInterfaceHost; @@ -330,8 +336,6 @@ impl super::Nexus { )) })?; - let vni: u32 = network_interface.vni.into(); - info!(log, "looking up instance's external IPs"; "instance_id" => %instance_id); @@ -349,6 +353,9 @@ impl super::Nexus { } } + let sled_address = + Ipv6Net(Ipv6Network::new(*sled_ip_address.ip(), 128).unwrap()); + for target_ip in ips .iter() .enumerate() @@ -361,29 +368,58 @@ impl super::Nexus { }) .map(|(_, ip)| ip) { - retry_until_known_result(log, || async { - dpd_client - .ensure_nat_entry( - &log, - target_ip.ip, - dpd_client::types::MacAddr { - a: mac_address.into_array(), - }, - *target_ip.first_port, - *target_ip.last_port, - vni, - sled_ip_address.ip(), - ) - .await - }) - .await - .map_err(|e| { - Error::internal_error(&format!( - "failed to ensure dpd entry: {e}" - )) - })?; + // For each external ip, add a nat entry to the database + self.ensure_nat_entry( + target_ip, + sled_address, + &network_interface, + mac_address, + opctx, + ) + .await?; } + // Notify dendrite that there are changes for it to reconcile. + // In the event of a failure to notify dendrite, we'll log an error + // and rely on dendrite's RPW timer to catch it up. + if let Err(e) = dpd_client.ipv4_nat_trigger_update().await { + error!(self.log, "failed to notify dendrite of nat updates"; "error" => ?e); + }; + + Ok(()) + } + + async fn ensure_nat_entry( + &self, + target_ip: &nexus_db_model::ExternalIp, + sled_address: Ipv6Net, + network_interface: &sled_agent_client::types::NetworkInterface, + mac_address: macaddr::MacAddr6, + opctx: &OpContext, + ) -> Result<(), Error> { + match target_ip.ip { + IpNetwork::V4(v4net) => { + let nat_entry = Ipv4NatValues { + external_address: Ipv4Net(v4net).into(), + first_port: target_ip.first_port, + last_port: target_ip.last_port, + sled_address: sled_address.into(), + vni: DbVni(network_interface.vni.clone().into()), + mac: nexus_db_model::MacAddr( + omicron_common::api::external::MacAddr(mac_address), + ), + }; + self.db_datastore + .ensure_ipv4_nat_entry(opctx, nat_entry) + .await?; + } + IpNetwork::V6(_v6net) => { + // TODO: implement handling of v6 nat. + return Err(Error::InternalError { + internal_message: "ipv6 nat is not yet implemented".into(), + }); + } + }; Ok(()) } @@ -419,55 +455,54 @@ impl super::Nexus { let mut errors = vec![]; for entry in external_ips { - for switch in &boundary_switches { - debug!(log, "deleting instance nat mapping"; - "instance_id" => %instance_id, - "switch" => switch.to_string(), - "entry" => #?entry); - - let client_result = - self.dpd_clients.get(switch).ok_or_else(|| { - Error::internal_error(&format!( - "unable to find dendrite client for {switch}" - )) - }); - - let dpd_client = match client_result { - Ok(client) => client, - Err(new_error) => { - errors.push(new_error); - continue; + // Soft delete the NAT entry + match self + .db_datastore + .ipv4_nat_delete_by_external_ip(&opctx, &entry) + .await + { + Ok(_) => Ok(()), + Err(err) => match err { + Error::ObjectNotFound { .. } => { + warn!(log, "no matching nat entries to soft delete"); + Ok(()) } - }; + _ => { + let message = format!( + "failed to delete nat entry due to error: {err:?}" + ); + error!(log, "{}", message); + Err(Error::internal_error(&message)) + } + }, + }?; + } - let result = retry_until_known_result(log, || async { - dpd_client - .ensure_nat_entry_deleted( - log, - entry.ip, - *entry.first_port, - ) - .await - }) - .await; - - if let Err(e) = result { - let e = Error::internal_error(&format!( - "failed to delete nat entry via dpd: {e}" - )); - - error!(log, "error deleting nat mapping: {e:#?}"; - "instance_id" => %instance_id, - "switch" => switch.to_string(), - "entry" => #?entry); - errors.push(e); - } else { - debug!(log, "deleting nat mapping successful"; - "instance_id" => %instance_id, - "switch" => switch.to_string(), - "entry" => #?entry); + for switch in &boundary_switches { + debug!(&self.log, "notifying dendrite of updates"; + "instance_id" => %authz_instance.id(), + "switch" => switch.to_string()); + + let client_result = self.dpd_clients.get(switch).ok_or_else(|| { + Error::internal_error(&format!( + "unable to find dendrite client for {switch}" + )) + }); + + let dpd_client = match client_result { + Ok(client) => client, + Err(new_error) => { + errors.push(new_error); + continue; } - } + }; + + // Notify dendrite that there are changes for it to reconcile. + // In the event of a failure to notify dendrite, we'll log an error + // and rely on dendrite's RPW timer to catch it up. + if let Err(e) = dpd_client.ipv4_nat_trigger_update().await { + error!(self.log, "failed to notify dendrite of nat updates"; "error" => ?e); + }; } if let Some(e) = errors.into_iter().nth(0) { @@ -496,32 +531,48 @@ impl super::Nexus { let boundary_switches = self.boundary_switches(opctx).await?; for external_ip in external_ips { - for switch in &boundary_switches { - debug!(&self.log, "deleting instance nat mapping"; + match self + .db_datastore + .ipv4_nat_delete_by_external_ip(&opctx, &external_ip) + .await + { + Ok(_) => Ok(()), + Err(err) => match err { + Error::ObjectNotFound { .. } => { + warn!( + self.log, + "no matching nat entries to soft delete" + ); + Ok(()) + } + _ => { + let message = format!( + "failed to delete nat entry due to error: {err:?}" + ); + error!(self.log, "{}", message); + Err(Error::internal_error(&message)) + } + }, + }?; + } + + for switch in &boundary_switches { + debug!(&self.log, "notifying dendrite of updates"; "instance_id" => %authz_instance.id(), - "switch" => switch.to_string(), - "entry" => #?external_ip); - - let dpd_client = - self.dpd_clients.get(switch).ok_or_else(|| { - Error::internal_error(&format!( - "unable to find dendrite client for {switch}" - )) - })?; - - dpd_client - .ensure_nat_entry_deleted( - &self.log, - external_ip.ip, - *external_ip.first_port, - ) - .await - .map_err(|e| { - Error::internal_error(&format!( - "failed to delete nat entry via dpd: {e}" - )) - })?; - } + "switch" => switch.to_string()); + + let dpd_client = self.dpd_clients.get(switch).ok_or_else(|| { + Error::internal_error(&format!( + "unable to find dendrite client for {switch}" + )) + })?; + + // Notify dendrite that there are changes for it to reconcile. + // In the event of a failure to notify dendrite, we'll log an error + // and rely on dendrite's RPW timer to catch it up. + if let Err(e) = dpd_client.ipv4_nat_trigger_update().await { + error!(self.log, "failed to notify dendrite of nat updates"; "error" => ?e); + }; } Ok(()) diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index ef8132451a..18c9dae841 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -349,6 +349,7 @@ impl Nexus { &background_ctx, Arc::clone(&db_datastore), &config.pkg.background_tasks, + &dpd_clients, config.deployment.id, resolver.clone(), ); diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs index ebb21feb40..9a20911893 100644 --- a/nexus/src/internal_api/http_entrypoints.rs +++ b/nexus/src/internal_api/http_entrypoints.rs @@ -24,6 +24,7 @@ use dropshot::RequestContext; use dropshot::ResultsPage; use dropshot::TypedBody; use hyper::Body; +use nexus_db_model::Ipv4NatEntryView; use nexus_types::internal_api::params::SwitchPutRequest; use nexus_types::internal_api::params::SwitchPutResponse; use nexus_types::internal_api::views::to_list; @@ -68,6 +69,8 @@ pub(crate) fn internal_api() -> NexusApiDescription { api.register(saga_list)?; api.register(saga_view)?; + api.register(ipv4_nat_changeset)?; + api.register(bgtask_list)?; api.register(bgtask_view)?; @@ -540,3 +543,51 @@ async fn bgtask_view( }; apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await } + +// NAT RPW internal APIs + +/// Path parameters for NAT ChangeSet +#[derive(Deserialize, JsonSchema)] +struct RpwNatPathParam { + /// which change number to start generating + /// the change set from + from_gen: i64, +} + +/// Query parameters for NAT ChangeSet +#[derive(Deserialize, JsonSchema)] +struct RpwNatQueryParam { + limit: u32, +} + +/// Fetch NAT ChangeSet +/// +/// Caller provides their generation as `from_gen`, along with a query +/// parameter for the page size (`limit`). Endpoint will return changes +/// that have occured since the caller's generation number up to the latest +/// change or until the `limit` is reached. If there are no changes, an +/// empty vec is returned. +#[endpoint { + method = GET, + path = "/nat/ipv4/changeset/{from_gen}" +}] +async fn ipv4_nat_changeset( + rqctx: RequestContext>, + path_params: Path, + query_params: Query, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = crate::context::op_context_for_internal_api(&rqctx).await; + let nexus = &apictx.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let mut changeset = nexus + .datastore() + .ipv4_nat_changeset(&opctx, path.from_gen, query.limit) + .await?; + changeset.sort_by_key(|e| e.gen); + Ok(HttpResponseOk(changeset)) + }; + apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await +} diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 54f7e03eef..fbed9aed8e 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -90,6 +90,7 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 +nat_cleanup.period_secs = 30 # How frequently to collect hardware/software inventory from the whole system # (even if we don't have reason to believe anything has changed). inventory.period_secs = 600 diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index f83cf68a8a..fcb285d9eb 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -323,6 +323,57 @@ } } }, + "/nat/ipv4/changeset/{from_gen}": { + "get": { + "summary": "Fetch NAT ChangeSet", + "description": "Caller provides their generation as `from_gen`, along with a query parameter for the page size (`limit`). Endpoint will return changes that have occured since the caller's generation number up to the latest change or until the `limit` is reached. If there are no changes, an empty vec is returned.", + "operationId": "ipv4_nat_changeset", + "parameters": [ + { + "in": "path", + "name": "from_gen", + "description": "which change number to start generating the change set from", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + }, + { + "in": "query", + "name": "limit", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Ipv4NatEntryView", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4NatEntryView" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/physical-disk": { "put": { "summary": "Report that a physical disk for the specified sled has come online.", @@ -3763,6 +3814,53 @@ } ] }, + "Ipv4NatEntryView": { + "description": "NAT Record", + "type": "object", + "properties": { + "deleted": { + "type": "boolean" + }, + "external_address": { + "type": "string", + "format": "ipv4" + }, + "first_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "gen": { + "type": "integer", + "format": "int64" + }, + "last_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "mac": { + "$ref": "#/components/schemas/MacAddr" + }, + "sled_address": { + "type": "string", + "format": "ipv6" + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "deleted", + "external_address", + "first_port", + "gen", + "last_port", + "mac", + "sled_address", + "vni" + ] + }, "Ipv4Network": { "type": "string", "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\/(3[0-2]|[0-2]?[0-9])$" @@ -5335,6 +5433,12 @@ "time_updated" ] }, + "Vni": { + "description": "A Geneve Virtual Network Identifier", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "ZpoolPutRequest": { "description": "Sent by a sled agent on startup to Nexus to request further instruction", "type": "object", diff --git a/package-manifest.toml b/package-manifest.toml index f320215a13..ca96341f2a 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -476,8 +476,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "147b03901aa8305b5271e0133a09f628b8140949" -source.sha256 = "14fe7f904f963b50188d6e060106b63df6d061ca64238f7b21623c432b5944e3" +source.commit = "8ff834e7d0a6adb263240edd40537f2c0768f1a4" +source.sha256 = "c00e79f55e0bdf048069b2d18a4d009ddfef46e7e5d846887cf96e843a8884bd" output.type = "zone" output.intermediate_only = true @@ -501,8 +501,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "147b03901aa8305b5271e0133a09f628b8140949" -source.sha256 = "f3aa685e4096f8f6e2ea6c169f391dbb88707abcbf1d2bde29163d81736e8ec6" +source.commit = "8ff834e7d0a6adb263240edd40537f2c0768f1a4" +source.sha256 = "428cce1e9aa399b1b49c04e7fd0bc1cb0e3f3fae6fda96055892a42e010c9d6f" output.type = "zone" output.intermediate_only = true @@ -519,8 +519,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "147b03901aa8305b5271e0133a09f628b8140949" -source.sha256 = "dece729ce4127216fba48e9cfed90ec2e5a57ee4ca6c4afc5fa770de6ea636bf" +source.commit = "8ff834e7d0a6adb263240edd40537f2c0768f1a4" +source.sha256 = "5dd3534bec5eb4f857d0bf3994b26650288f650d409eec6aaa29860a2f481c37" output.type = "zone" output.intermediate_only = true diff --git a/schema/crdb/10.0.0/README.md b/schema/crdb/11.0.0/README.md similarity index 100% rename from schema/crdb/10.0.0/README.md rename to schema/crdb/11.0.0/README.md diff --git a/schema/crdb/10.0.0/up01.sql b/schema/crdb/11.0.0/up01.sql similarity index 100% rename from schema/crdb/10.0.0/up01.sql rename to schema/crdb/11.0.0/up01.sql diff --git a/schema/crdb/10.0.0/up02.sql b/schema/crdb/11.0.0/up02.sql similarity index 100% rename from schema/crdb/10.0.0/up02.sql rename to schema/crdb/11.0.0/up02.sql diff --git a/schema/crdb/10.0.0/up03.sql b/schema/crdb/11.0.0/up03.sql similarity index 100% rename from schema/crdb/10.0.0/up03.sql rename to schema/crdb/11.0.0/up03.sql diff --git a/schema/crdb/10.0.0/up04.sql b/schema/crdb/11.0.0/up04.sql similarity index 100% rename from schema/crdb/10.0.0/up04.sql rename to schema/crdb/11.0.0/up04.sql diff --git a/schema/crdb/10.0.0/up05.sql b/schema/crdb/11.0.0/up05.sql similarity index 100% rename from schema/crdb/10.0.0/up05.sql rename to schema/crdb/11.0.0/up05.sql diff --git a/schema/crdb/10.0.0/up06.sql b/schema/crdb/11.0.0/up06.sql similarity index 100% rename from schema/crdb/10.0.0/up06.sql rename to schema/crdb/11.0.0/up06.sql diff --git a/schema/crdb/10.0.0/up07.sql b/schema/crdb/11.0.0/up07.sql similarity index 100% rename from schema/crdb/10.0.0/up07.sql rename to schema/crdb/11.0.0/up07.sql diff --git a/schema/crdb/10.0.0/up08.sql b/schema/crdb/11.0.0/up08.sql similarity index 100% rename from schema/crdb/10.0.0/up08.sql rename to schema/crdb/11.0.0/up08.sql diff --git a/schema/crdb/10.0.0/up09.sql b/schema/crdb/11.0.0/up09.sql similarity index 100% rename from schema/crdb/10.0.0/up09.sql rename to schema/crdb/11.0.0/up09.sql diff --git a/schema/crdb/11.0.0/up1.sql b/schema/crdb/11.0.0/up1.sql new file mode 100644 index 0000000000..a4d31edd71 --- /dev/null +++ b/schema/crdb/11.0.0/up1.sql @@ -0,0 +1 @@ +CREATE SEQUENCE IF NOT EXISTS omicron.public.ipv4_nat_version START 1 INCREMENT 1; diff --git a/schema/crdb/10.0.0/up10.sql b/schema/crdb/11.0.0/up10.sql similarity index 100% rename from schema/crdb/10.0.0/up10.sql rename to schema/crdb/11.0.0/up10.sql diff --git a/schema/crdb/10.0.0/up11.sql b/schema/crdb/11.0.0/up11.sql similarity index 100% rename from schema/crdb/10.0.0/up11.sql rename to schema/crdb/11.0.0/up11.sql diff --git a/schema/crdb/10.0.0/up12.sql b/schema/crdb/11.0.0/up12.sql similarity index 100% rename from schema/crdb/10.0.0/up12.sql rename to schema/crdb/11.0.0/up12.sql diff --git a/schema/crdb/11.0.0/up2.sql b/schema/crdb/11.0.0/up2.sql new file mode 100644 index 0000000000..b92d4c73d3 --- /dev/null +++ b/schema/crdb/11.0.0/up2.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.ipv4_nat_entry ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_address INET NOT NULL, + first_port INT4 NOT NULL, + last_port INT4 NOT NULL, + sled_address INET NOT NULL, + vni INT4 NOT NULL, + mac INT8 NOT NULL, + version_added INT8 NOT NULL DEFAULT nextval('omicron.public.ipv4_nat_version'), + version_removed INT8, + time_created TIMESTAMPTZ NOT NULL DEFAULT now(), + time_deleted TIMESTAMPTZ +); diff --git a/schema/crdb/11.0.0/up3.sql b/schema/crdb/11.0.0/up3.sql new file mode 100644 index 0000000000..1247aad693 --- /dev/null +++ b/schema/crdb/11.0.0/up3.sql @@ -0,0 +1,13 @@ +CREATE UNIQUE INDEX IF NOT EXISTS ipv4_nat_version_added ON omicron.public.ipv4_nat_entry ( + version_added +) +STORING ( + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + time_created, + time_deleted +); diff --git a/schema/crdb/11.0.0/up4.sql b/schema/crdb/11.0.0/up4.sql new file mode 100644 index 0000000000..b9cfe305d2 --- /dev/null +++ b/schema/crdb/11.0.0/up4.sql @@ -0,0 +1,5 @@ +CREATE UNIQUE INDEX IF NOT EXISTS overlapping_ipv4_nat_entry ON omicron.public.ipv4_nat_entry ( + external_address, + first_port, + last_port +) WHERE time_deleted IS NULL; diff --git a/schema/crdb/11.0.0/up5.sql b/schema/crdb/11.0.0/up5.sql new file mode 100644 index 0000000000..dce2211eae --- /dev/null +++ b/schema/crdb/11.0.0/up5.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS ipv4_nat_lookup ON omicron.public.ipv4_nat_entry (external_address, first_port, last_port, sled_address, vni, mac); diff --git a/schema/crdb/11.0.0/up6.sql b/schema/crdb/11.0.0/up6.sql new file mode 100644 index 0000000000..e4958eb352 --- /dev/null +++ b/schema/crdb/11.0.0/up6.sql @@ -0,0 +1,13 @@ +CREATE UNIQUE INDEX IF NOT EXISTS ipv4_nat_version_removed ON omicron.public.ipv4_nat_entry ( + version_removed +) +STORING ( + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + time_created, + time_deleted +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 875877ee96..a74cabfe6e 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -2738,12 +2738,24 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_caboose ( COMMIT; BEGIN; -/*******************************************************************/ +CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( + -- There should only be one row of this table for the whole DB. + -- It's a little goofy, but filter on "singleton = true" before querying + -- or applying updates, and you'll access the singleton row. + -- + -- We also add a constraint on this table to ensure it's not possible to + -- access the version of this table with "singleton = false". + singleton BOOL NOT NULL PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + -- Semver representation of the DB version + version STRING(64) NOT NULL, -/* - * Metadata for the schema itself. This version number isn't great, as there's - * nothing to ensure it gets bumped when it should be, but it's a start. - */ + -- (Optional) Semver representation of the DB version to which we're upgrading + target_version STRING(64), + + CHECK (singleton = true) +); -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( @@ -2812,6 +2824,62 @@ CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; +CREATE SEQUENCE IF NOT EXISTS omicron.public.ipv4_nat_version START 1 INCREMENT 1; + +CREATE TABLE IF NOT EXISTS omicron.public.ipv4_nat_entry ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_address INET NOT NULL, + first_port INT4 NOT NULL, + last_port INT4 NOT NULL, + sled_address INET NOT NULL, + vni INT4 NOT NULL, + mac INT8 NOT NULL, + version_added INT8 NOT NULL DEFAULT nextval('omicron.public.ipv4_nat_version'), + version_removed INT8, + time_created TIMESTAMPTZ NOT NULL DEFAULT now(), + time_deleted TIMESTAMPTZ +); + +CREATE UNIQUE INDEX IF NOT EXISTS ipv4_nat_version_added ON omicron.public.ipv4_nat_entry ( + version_added +) +STORING ( + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + time_created, + time_deleted +); + +CREATE UNIQUE INDEX IF NOT EXISTS overlapping_ipv4_nat_entry ON omicron.public.ipv4_nat_entry ( + external_address, + first_port, + last_port +) WHERE time_deleted IS NULL; + +CREATE INDEX IF NOT EXISTS ipv4_nat_lookup ON omicron.public.ipv4_nat_entry (external_address, first_port, last_port, sled_address, vni, mac); + +CREATE UNIQUE INDEX IF NOT EXISTS ipv4_nat_version_removed ON omicron.public.ipv4_nat_entry ( + version_removed +) +STORING ( + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + time_created, + time_deleted +); + +/* + * Metadata for the schema itself. This version number isn't great, as there's + * nothing to ensure it gets bumped when it should be, but it's a start. + */ CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( -- There should only be one row of this table for the whole DB. -- It's a little goofy, but filter on "singleton = true" before querying @@ -2838,7 +2906,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '10.0.0', NULL) + ( TRUE, NOW(), NOW(), '11.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index cae1f650c9..94c8f5572e 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -38,6 +38,7 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 +nat_cleanup.period_secs = 30 # How frequently to collect hardware/software inventory from the whole system # (even if we don't have reason to believe anything has changed). inventory.period_secs = 600 diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index be8683be54..fcaa6176a8 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -38,6 +38,7 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 +nat_cleanup.period_secs = 30 # How frequently to collect hardware/software inventory from the whole system # (even if we don't have reason to believe anything has changed). inventory.period_secs = 600 diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index aadf68da1b..ba4b5a5722 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="147b03901aa8305b5271e0133a09f628b8140949" -SHA2="82437c74afd4894aa5b9ea800d5777793e8777fe87471321dd22ad1a1c9c9ef3" +COMMIT="8ff834e7d0a6adb263240edd40537f2c0768f1a4" +SHA2="07d115bfa8498a8015ca2a8447efeeac32e24aeb25baf3d5e2313216e11293c0" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index 81a957323c..619a6bf287 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="14fe7f904f963b50188d6e060106b63df6d061ca64238f7b21623c432b5944e3" -CIDL_SHA256_LINUX_DPD="fff6c7484bbb06aa644e3fe41b200e4f7f8d7f65d067cbecd851c834c15fe2ec" -CIDL_SHA256_LINUX_SWADM="0449383a57468aec3b5a4ad26962cfc9e9a121bd13e777329e8a70767e6d9aae" +CIDL_SHA256_ILLUMOS="c00e79f55e0bdf048069b2d18a4d009ddfef46e7e5d846887cf96e843a8884bd" +CIDL_SHA256_LINUX_DPD="b5d829b4628759ac374106f3c56c29074b29577fd0ff72f61c3b8289fea430fe" +CIDL_SHA256_LINUX_SWADM="afc68828f54dc57b32dc1556fc588baeab12341c30e96cc0fadb49f401b4b48f"