diff --git a/.github/workflows/check-opte-ver.yml b/.github/workflows/check-opte-ver.yml index 42ef1dda11..a8c0febc2d 100644 --- a/.github/workflows/check-opte-ver.yml +++ b/.github/workflows/check-opte-ver.yml @@ -9,7 +9,7 @@ jobs: check-opte-ver: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - name: Install jq diff --git a/.github/workflows/check-workspace-deps.yml b/.github/workflows/check-workspace-deps.yml index f94ed32fde..ec2bcc3537 100644 --- a/.github/workflows/check-workspace-deps.yml +++ b/.github/workflows/check-workspace-deps.yml @@ -10,7 +10,7 @@ jobs: check-workspace-deps: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - name: Check Workspace Dependencies diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index 07b7124f73..cc67b91fce 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -24,7 +24,7 @@ jobs: with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@7c4edf14345f90e1199544e41cb94c3ef67bd237 # v2 + uses: taiki-e/install-action@ccc14bdc8d34cddf54e4f9fb2da0c208427207a3 # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6239add88f..23ccc7e61f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -9,7 +9,7 @@ jobs: check-style: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - name: Report cargo version @@ -31,7 +31,7 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1 @@ -61,7 +61,7 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1 @@ -91,7 +91,7 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1 diff --git a/.github/workflows/update-dendrite.yml b/.github/workflows/update-dendrite.yml index 9d79dfc8f9..4a2e6e95cc 100644 --- a/.github/workflows/update-dendrite.yml +++ b/.github/workflows/update-dendrite.yml @@ -29,7 +29,7 @@ jobs: steps: # Checkout both the target and integration branches - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: token: ${{ inputs.reflector_access_token }} fetch-depth: 0 @@ -39,7 +39,7 @@ jobs: . ./tools/reflector/helpers.sh PATHS=("tools") - merge $TARGET_BRANCH $INT_BRANCH ${{ inputs.reflector_user_id }} $PATHS + merge $TARGET_BRANCH $INT_BRANCH ${{ inputs.reflector_user_id }} "${PATHS[@]}" - name: Update dendrite versions run: | diff --git a/.github/workflows/update-maghemite.yml b/.github/workflows/update-maghemite.yml index e2512dc6ce..b3611f9987 100644 --- a/.github/workflows/update-maghemite.yml +++ b/.github/workflows/update-maghemite.yml @@ -29,7 +29,7 @@ jobs: steps: # Checkout both the target and integration branches - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: token: ${{ inputs.reflector_access_token }} fetch-depth: 0 @@ -39,7 +39,7 @@ jobs: . ./tools/reflector/helpers.sh PATHS=("tools") - merge $TARGET_BRANCH $INT_BRANCH ${{ inputs.reflector_user_id }} $PATHS + merge $TARGET_BRANCH $INT_BRANCH ${{ inputs.reflector_user_id }} "${PATHS[@]}" - name: Update maghemite versions run: | diff --git a/.github/workflows/validate-openapi-spec.yml b/.github/workflows/validate-openapi-spec.yml index ea77ed9497..39c6c1debb 100644 --- a/.github/workflows/validate-openapi-spec.yml +++ b/.github/workflows/validate-openapi-spec.yml @@ -10,7 +10,7 @@ jobs: format: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2 diff --git a/Cargo.lock b/Cargo.lock index 77699ee0bc..3498d92e9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1059,9 +1059,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "cookie" -version = "0.16.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" +checksum = "3cd91cf61412820176e137621345ee43b3f4423e589e7ae4e50d601d93e35ef8" dependencies = [ "time", "version_check", @@ -1640,9 +1640,9 @@ dependencies = [ [[package]] name = "diesel" -version = "2.1.3" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2268a214a6f118fce1838edba3d1561cf0e78d8de785475957a580a7f8c69d33" +checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8" dependencies = [ "bitflags 2.4.0", "byteorder", @@ -2291,9 +2291,12 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" [[package]] name = "fs-err" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" +checksum = "fb5fd9bcbe8b1087cbd395b51498c01bc997cef73e778a80b77a811af5e2d29f" +dependencies = [ + "autocfg", +] [[package]] name = "fs2" @@ -2798,9 +2801,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -3183,7 +3186,7 @@ dependencies = [ "installinator-artifact-client", "installinator-common", "ipcc-key-value", - "itertools 0.11.0", + "itertools 0.12.0", "libc", "omicron-common", "omicron-test-utils", @@ -3408,6 +3411,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -4010,7 +4022,7 @@ dependencies = [ "hyper-rustls", "internal-dns", "ipnetwork", - "itertools 0.11.0", + "itertools 0.12.0", "lazy_static", "macaddr", "newtype_derive", @@ -4604,7 +4616,7 @@ dependencies = [ "hyper-rustls", "internal-dns", "ipnetwork", - "itertools 0.11.0", + "itertools 0.12.0", "lazy_static", "macaddr", "mg-admin-client", @@ -4684,6 +4696,7 @@ dependencies = [ "async-bb8-diesel", "chrono", "clap 4.4.3", + "crucible-agent-client", "diesel", "dropshot", "expectorate", @@ -4811,7 +4824,7 @@ dependencies = [ "illumos-utils", "internal-dns", "ipnetwork", - "itertools 0.11.0", + "itertools 0.12.0", "key-manager", "libc", "macaddr", @@ -5002,9 +5015,9 @@ dependencies = [ [[package]] name = "omicron-zone-package" -version = "0.8.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dc0973625837d1c4e31d4aa60e72008f3af3aa9b0d0ebfd5b5dc67d2e721a48" +checksum = "620c53207d39a385f298444337d575690e0d9e793561d471ba7a614dc213e372" dependencies = [ "anyhow", "async-trait", @@ -5012,7 +5025,9 @@ dependencies = [ "filetime", "flate2", "futures-util", + "hex", "reqwest", + "ring 0.16.20", "semver 1.0.20", "serde", "serde_derive", @@ -5326,7 +5341,7 @@ dependencies = [ "dropshot", "expectorate", "highway", - "itertools 0.11.0", + "itertools 0.12.0", "omicron-common", "omicron-test-utils", "omicron-workspace-hack", @@ -6150,9 +6165,9 @@ dependencies = [ [[package]] name = "proptest" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c003ac8c77cb07bb74f5f198bce836a689bcd5a42574612bf14d17bfd08c20e" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" dependencies = [ "bit-set", "bit-vec", @@ -6162,7 +6177,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", "rusty-fork", "tempfile", "unarray", @@ -6868,9 +6883,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.8" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", "ring 0.17.5", @@ -8855,7 +8870,7 @@ dependencies = [ "futures", "hex", "hubtools", - "itertools 0.11.0", + "itertools 0.12.0", "omicron-common", "omicron-test-utils", "omicron-workspace-hack", @@ -9435,7 +9450,7 @@ dependencies = [ "humantime", "indexmap 2.1.0", "indicatif", - "itertools 0.11.0", + "itertools 0.12.0", "omicron-common", "omicron-passwords", "omicron-workspace-hack", @@ -9541,7 +9556,7 @@ dependencies = [ "installinator-common", "internal-dns", "ipnetwork", - "itertools 0.11.0", + "itertools 0.12.0", "maplit", "omicron-certificates", "omicron-common", diff --git a/Cargo.toml b/Cargo.toml index f3aa814269..b18b20aec7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -165,7 +165,7 @@ ciborium = "0.2.1" cfg-if = "1.0" chrono = { version = "0.4", features = [ "serde" ] } clap = { version = "4.4", features = ["derive", "env", "wrap_help"] } -cookie = "0.16" +cookie = "0.18" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } @@ -180,7 +180,7 @@ db-macros = { path = "nexus/db-macros" } debug-ignore = "1.0.5" derive_more = "0.99.17" derive-where = "1.2.5" -diesel = { version = "2.1.3", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } +diesel = { version = "2.1.4", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } diesel-dtrace = { git = "https://github.com/oxidecomputer/diesel-dtrace", branch = "main" } dns-server = { path = "dns-server" } dns-service-client = { path = "clients/dns-service-client" } @@ -193,7 +193,7 @@ filetime = "0.2.22" flate2 = "1.0.28" flume = "0.11.0" foreign-types = "0.3.2" -fs-err = "2.9.0" +fs-err = "2.10.0" futures = "0.3.29" gateway-client = { path = "clients/gateway-client" } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", default-features = false, features = ["std"] } @@ -206,7 +206,7 @@ hex = "0.4.3" hex-literal = "0.4.1" highway = "1.1.0" hkdf = "0.12.3" -http = "0.2.9" +http = "0.2.11" httptest = "0.15.5" hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } humantime = "2.1.0" @@ -223,7 +223,7 @@ installinator-common = { path = "installinator-common" } internal-dns = { path = "internal-dns" } ipcc-key-value = { path = "ipcc-key-value" } ipnetwork = { version = "0.20", features = ["schemars"] } -itertools = "0.11.0" +itertools = "0.12.0" key-manager = { path = "key-manager" } kstat-rs = "0.2.3" lazy_static = "1.4.0" @@ -256,7 +256,7 @@ omicron-package = { path = "package" } omicron-rpaths = { path = "rpaths" } omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } -omicron-zone-package = "0.8.3" +omicron-zone-package = "0.9.1" oxide-client = { path = "clients/oxide-client" } oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649", features = [ "api", "std" ] } once_cell = "1.18.0" @@ -293,7 +293,7 @@ progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branc bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" } propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" } propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" } -proptest = "1.3.1" +proptest = "1.4.0" quote = "1.0" rand = "0.8.5" ratatui = "0.23.0" @@ -307,7 +307,7 @@ ring = "0.16" rpassword = "7.2.0" rstest = "0.18.2" rustfmt-wrapper = "0.2" -rustls = "0.21.8" +rustls = "0.21.9" samael = { git = "https://github.com/njaremko/samael", features = ["xmlsec"], branch = "master" } schemars = "0.8.12" secrecy = "0.8.0" diff --git a/bootstore/Cargo.toml b/bootstore/Cargo.toml index 18e3e3876b..93eb6a3c48 100644 --- a/bootstore/Cargo.toml +++ b/bootstore/Cargo.toml @@ -27,7 +27,7 @@ slog.workspace = true thiserror.workspace = true tokio.workspace = true uuid.workspace = true -vsss-rs = { version = "3.2.0", features = ["std", "curve25519"] } +vsss-rs = { version = "3.3.1", features = ["std", "curve25519"] } zeroize.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index fcea57220d..adf661516a 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -750,6 +750,7 @@ pub enum ResourceType { UserBuiltin, Zpool, Vmm, + Ipv4NatEntry, } // IDENTITY METADATA diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index 4e821e2676..94c39b4436 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -335,6 +335,8 @@ pub struct BackgroundTaskConfig { pub dns_external: DnsTasksConfig, /// configuration for external endpoint list watcher pub external_endpoints: ExternalEndpointsConfig, + /// configuration for nat table garbage collector + pub nat_cleanup: NatCleanupConfig, /// configuration for inventory tasks pub inventory: InventoryConfig, } @@ -371,6 +373,14 @@ pub struct ExternalEndpointsConfig { // allow/disallow wildcard certs, don't serve expired certs, etc.) } +#[serde_as] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct NatCleanupConfig { + /// period (in seconds) for periodic activations of this background task + #[serde_as(as = "DurationSeconds")] + pub period_secs: Duration, +} + #[serde_as] #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct InventoryConfig { @@ -498,7 +508,7 @@ mod test { BackgroundTaskConfig, Config, ConfigDropshotWithTls, ConsoleConfig, Database, DeploymentConfig, DnsTasksConfig, DpdConfig, ExternalEndpointsConfig, InternalDns, InventoryConfig, LoadError, - LoadErrorKind, MgdConfig, PackageConfig, SchemeName, + LoadErrorKind, MgdConfig, NatCleanupConfig, PackageConfig, SchemeName, TimeseriesDbConfig, Tunables, UpdatesConfig, }; use crate::address::{Ipv6Subnet, RACK_PREFIX}; @@ -649,6 +659,7 @@ mod test { dns_external.period_secs_propagation = 7 dns_external.max_concurrent_server_updates = 8 external_endpoints.period_secs = 9 + nat_cleanup.period_secs = 30 inventory.period_secs = 10 inventory.nkeep = 11 inventory.disable = false @@ -746,6 +757,9 @@ mod test { external_endpoints: ExternalEndpointsConfig { period_secs: Duration::from_secs(9), }, + nat_cleanup: NatCleanupConfig { + period_secs: Duration::from_secs(30), + }, inventory: InventoryConfig { period_secs: Duration::from_secs(10), nkeep: 11, @@ -804,6 +818,7 @@ mod test { dns_external.period_secs_propagation = 7 dns_external.max_concurrent_server_updates = 8 external_endpoints.period_secs = 9 + nat_cleanup.period_secs = 30 inventory.period_secs = 10 inventory.nkeep = 3 inventory.disable = false diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml index ff3c650d6d..a8834a0b29 100644 --- a/dev-tools/omdb/Cargo.toml +++ b/dev-tools/omdb/Cargo.toml @@ -12,6 +12,7 @@ anyhow.workspace = true async-bb8-diesel.workspace = true chrono.workspace = true clap.workspace = true +crucible-agent-client.workspace = true diesel.workspace = true dropshot.workspace = true futures.workspace = true @@ -39,10 +40,10 @@ tokio = { workspace = true, features = [ "full" ] } uuid.workspace = true ipnetwork.workspace = true omicron-workspace-hack.workspace = true +nexus-test-utils.workspace = true [dev-dependencies] expectorate.workspace = true -nexus-test-utils.workspace = true nexus-test-utils-macros.workspace = true omicron-nexus.workspace = true omicron-test-utils.workspace = true diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index efcefdea43..5fa19a1a27 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -19,7 +19,9 @@ use crate::Omdb; use anyhow::anyhow; use anyhow::bail; use anyhow::Context; +use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; +use async_bb8_diesel::AsyncSimpleConnection; use chrono::SecondsFormat; use clap::Args; use clap::Subcommand; @@ -30,6 +32,7 @@ use diesel::BoolExpressionMethods; use diesel::ExpressionMethods; use diesel::JoinOnDsl; use diesel::NullableExpressionMethods; +use diesel::TextExpressionMethods; use gateway_client::types::SpType; use nexus_db_model::Dataset; use nexus_db_model::Disk; @@ -49,16 +52,19 @@ use nexus_db_model::Snapshot; use nexus_db_model::SnapshotState; use nexus_db_model::SwCaboose; use nexus_db_model::Vmm; +use nexus_db_model::Volume; use nexus_db_model::Zpool; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; +use nexus_db_queries::db::datastore::read_only_resources_associated_with_volume; +use nexus_db_queries::db::datastore::CrucibleTargets; use nexus_db_queries::db::datastore::DataStoreConnection; -use nexus_db_queries::db::datastore::DataStoreInventoryTest; use nexus_db_queries::db::datastore::InstanceAndActiveVmm; use nexus_db_queries::db::identity::Asset; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::ServiceKind; use nexus_db_queries::db::DataStore; +use nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL; use nexus_types::identity::Resource; use nexus_types::internal_api::params::DnsRecord; use nexus_types::internal_api::params::Srv; @@ -67,6 +73,7 @@ use nexus_types::inventory::Collection; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Generation; use omicron_common::postgres_config::PostgresConfigWithUrl; +use sled_agent_client::types::VolumeConstructionRequest; use std::cmp::Ordering; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -126,7 +133,8 @@ pub struct DbArgs { /// limit to apply to queries that fetch rows #[clap( long = "fetch-limit", - default_value_t = NonZeroU32::new(500).unwrap() + default_value_t = NonZeroU32::new(500).unwrap(), + env("OMDB_FETCH_LIMIT"), )] fetch_limit: NonZeroU32, @@ -153,6 +161,8 @@ enum DbCommands { Network(NetworkArgs), /// Print information about snapshots Snapshots(SnapshotArgs), + /// Validate the contents of the database + Validate(ValidateArgs), } #[derive(Debug, Args)] @@ -309,6 +319,23 @@ struct SnapshotInfoArgs { uuid: Uuid, } +#[derive(Debug, Args)] +struct ValidateArgs { + #[command(subcommand)] + command: ValidateCommands, +} + +#[derive(Debug, Subcommand)] +enum ValidateCommands { + /// Validate each `volume_references` column in the region snapshots table + ValidateVolumeReferences, + + /// Find either region snapshots Nexus knows about that the corresponding + /// Crucible agent says were deleted, or region snapshots that Nexus doesn't + /// know about. + ValidateRegionSnapshots, +} + impl DbArgs { /// Run a `omdb db` subcommand. pub(crate) async fn run_cmd( @@ -383,8 +410,13 @@ impl DbArgs { .await } DbCommands::Inventory(inventory_args) => { - cmd_db_inventory(&datastore, self.fetch_limit, inventory_args) - .await + cmd_db_inventory( + &opctx, + &datastore, + self.fetch_limit, + inventory_args, + ) + .await } DbCommands::Services(ServicesArgs { command: ServicesCommands::ListInstances, @@ -425,6 +457,18 @@ impl DbArgs { DbCommands::Snapshots(SnapshotArgs { command: SnapshotCommands::List, }) => cmd_db_snapshot_list(&datastore, self.fetch_limit).await, + DbCommands::Validate(ValidateArgs { + command: ValidateCommands::ValidateVolumeReferences, + }) => { + cmd_db_validate_volume_references(&datastore, self.fetch_limit) + .await + } + DbCommands::Validate(ValidateArgs { + command: ValidateCommands::ValidateRegionSnapshots, + }) => { + cmd_db_validate_region_snapshots(&datastore, self.fetch_limit) + .await + } } } } @@ -1701,6 +1745,427 @@ async fn cmd_db_eips( Ok(()) } +/// Validate the `volume_references` column of the region snapshots table +async fn cmd_db_validate_volume_references( + datastore: &DataStore, + limit: NonZeroU32, +) -> Result<(), anyhow::Error> { + // First, get all region snapshot records + let region_snapshots: Vec = { + let region_snapshots: Vec = datastore + .pool_connection_for_tests() + .await? + .transaction_async(|conn| async move { + // Selecting all region snapshots requires a full table scan + conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL).await?; + + use db::schema::region_snapshot::dsl; + dsl::region_snapshot + .select(RegionSnapshot::as_select()) + .get_results_async(&conn) + .await + }) + .await?; + + check_limit(®ion_snapshots, limit, || { + String::from("listing region snapshots") + }); + + region_snapshots + }; + + #[derive(Tabled)] + struct Row { + dataset_id: Uuid, + region_id: Uuid, + snapshot_id: Uuid, + error: String, + } + + let mut rows = Vec::new(); + + // Then, for each, make sure that the `volume_references` matches what is in + // the volume table + for region_snapshot in region_snapshots { + let matching_volumes: Vec = { + let snapshot_addr = region_snapshot.snapshot_addr.clone(); + + let matching_volumes = datastore + .pool_connection_for_tests() + .await? + .transaction_async(|conn| async move { + // Selecting all volumes based on the data column requires a + // full table scan + conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL).await?; + + let pattern = format!("%{}%", &snapshot_addr); + + use db::schema::volume::dsl; + + // Find all volumes that have not been deleted that contain + // this snapshot_addr. If a Volume has been soft deleted, + // then the region snapshot record should have had its + // volume references column updated accordingly. + dsl::volume + .filter(dsl::time_deleted.is_null()) + .filter(dsl::data.like(pattern)) + .select(Volume::as_select()) + .get_results_async(&conn) + .await + }) + .await?; + + check_limit(&matching_volumes, limit, || { + String::from("finding matching volumes") + }); + + matching_volumes + }; + + // The Crucible Agent will reuse ports for regions and running snapshots + // when they're deleted. Check that the matching volume construction requests + // reference this snapshot addr as a read-only target. + let matching_volumes = matching_volumes + .into_iter() + .filter(|volume| { + let vcr: VolumeConstructionRequest = + serde_json::from_str(&volume.data()).unwrap(); + + let mut targets = CrucibleTargets::default(); + read_only_resources_associated_with_volume(&vcr, &mut targets); + + targets + .read_only_targets + .contains(®ion_snapshot.snapshot_addr) + }) + .count(); + + if matching_volumes != region_snapshot.volume_references as usize { + rows.push(Row { + dataset_id: region_snapshot.dataset_id, + region_id: region_snapshot.region_id, + snapshot_id: region_snapshot.snapshot_id, + error: format!( + "record has {} volume references when it should be {}!", + region_snapshot.volume_references, matching_volumes, + ), + }); + } else { + // The volume references are correct, but additionally check to see + // deleting is true when matching_volumes is 0. Be careful: in the + // snapshot create saga, the region snapshot record is created + // before the snapshot's volume is inserted into the DB. There's a + // time between these operations that this function would flag that + // this region snapshot should have `deleting` set to true. + + if matching_volumes == 0 && !region_snapshot.deleting { + rows.push(Row { + dataset_id: region_snapshot.dataset_id, + region_id: region_snapshot.region_id, + snapshot_id: region_snapshot.snapshot_id, + error: String::from( + "record has 0 volume references but deleting is false!", + ), + }); + } + } + } + + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .to_string(); + + println!("{}", table); + + Ok(()) +} + +async fn cmd_db_validate_region_snapshots( + datastore: &DataStore, + limit: NonZeroU32, +) -> Result<(), anyhow::Error> { + let mut regions_to_snapshots_map: BTreeMap> = + BTreeMap::default(); + + // First, get all region snapshot records (with their corresponding dataset) + let datasets_and_region_snapshots: Vec<(Dataset, RegionSnapshot)> = { + let datasets_region_snapshots: Vec<(Dataset, RegionSnapshot)> = + datastore + .pool_connection_for_tests() + .await? + .transaction_async(|conn| async move { + // Selecting all datasets and region snapshots requires a full table scan + conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL).await?; + + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region_snapshot::dsl; + + dsl::region_snapshot + .inner_join( + dataset_dsl::dataset + .on(dsl::dataset_id.eq(dataset_dsl::id)), + ) + .select(( + Dataset::as_select(), + RegionSnapshot::as_select(), + )) + .get_results_async(&conn) + .await + }) + .await?; + + check_limit(&datasets_region_snapshots, limit, || { + String::from("listing datasets and region snapshots") + }); + + datasets_region_snapshots + }; + + #[derive(Tabled)] + struct Row { + dataset_id: Uuid, + region_id: Uuid, + snapshot_id: Uuid, + dataset_addr: std::net::SocketAddrV6, + error: String, + } + + let mut rows = Vec::new(); + + // Then, for each one, reconcile with the corresponding Crucible Agent: do + // the region_snapshot records match reality? + for (dataset, region_snapshot) in datasets_and_region_snapshots { + regions_to_snapshots_map + .entry(region_snapshot.region_id) + .or_default() + .insert(region_snapshot.snapshot_id); + + use crucible_agent_client::types::RegionId; + use crucible_agent_client::types::State; + use crucible_agent_client::Client as CrucibleAgentClient; + + let url = format!("http://{}", dataset.address()); + let client = CrucibleAgentClient::new(&url); + + let actual_region_snapshots = client + .region_get_snapshots(&RegionId( + region_snapshot.region_id.to_string(), + )) + .await?; + + let snapshot_id = region_snapshot.snapshot_id.to_string(); + + if actual_region_snapshots + .snapshots + .iter() + .any(|x| x.name == snapshot_id) + { + // A snapshot currently exists, matching the database entry + } else { + // In this branch, there's a database entry for a snapshot that was + // deleted. Due to how the snapshot create saga is currently + // written, a database entry would not have been created unless a + // snapshot was successfully made: unless that saga changes, we can + // be reasonably sure that this snapshot existed at some point. + + match actual_region_snapshots.running_snapshots.get(&snapshot_id) { + Some(running_snapshot) => { + match running_snapshot.state { + State::Destroyed | State::Failed => { + // In this branch, we can be sure a snapshot previously + // existed and was deleted: a running snapshot was made + // from it, then deleted, and the snapshot does not + // currently exist in the list of snapshots for this + // region. This record should be deleted. + + // Before recommending anything, validate the higher + // level Snapshot object too: it should have been + // destroyed. + + let snapshot: Snapshot = { + use db::schema::snapshot::dsl; + + dsl::snapshot + .filter( + dsl::id.eq(region_snapshot.snapshot_id), + ) + .select(Snapshot::as_select()) + .first_async( + &*datastore + .pool_connection_for_tests() + .await?, + ) + .await? + }; + + if snapshot.time_deleted().is_some() { + // This is ok - Nexus currently soft-deletes its + // resource records. + rows.push(Row { + dataset_id: region_snapshot.dataset_id, + region_id: region_snapshot.region_id, + snapshot_id: region_snapshot.snapshot_id, + dataset_addr: dataset.address(), + error: String::from( + "region snapshot was deleted, please remove its record", + ), + }); + } else { + // If the higher level Snapshot was _not_ + // deleted, this is a Nexus bug: something told + // the Agent to delete the snapshot when the + // higher level Snapshot was not deleted! + + rows.push(Row { + dataset_id: region_snapshot.dataset_id, + region_id: region_snapshot.region_id, + snapshot_id: region_snapshot.snapshot_id, + dataset_addr: dataset.address(), + error: String::from( + "NEXUS BUG: region snapshot was deleted, but the higher level snapshot was not!", + ), + }); + } + } + + State::Requested + | State::Created + | State::Tombstoned => { + // The agent is in a bad state: we did not find the + // snapshot in the list of snapshots for this + // region, but either: + // + // - there's a requested or existing running + // snapshot for it, or + // + // - there's a running snapshot that should have + // been completely deleted before the snapshot + // itself was deleted. + // + // This should have never been allowed to happen by + // the Agent, so it's a bug. + + rows.push(Row { + dataset_id: region_snapshot.dataset_id, + region_id: region_snapshot.region_id, + snapshot_id: region_snapshot.snapshot_id, + dataset_addr: dataset.address(), + error: format!( + "AGENT BUG: region snapshot was deleted but has a running snapshot in state {:?}!", + running_snapshot.state, + ), + }); + } + } + } + + None => { + // A running snapshot never existed for this snapshot + } + } + } + } + + // Second, get all regions + let datasets_and_regions: Vec<(Dataset, Region)> = { + let datasets_and_regions: Vec<(Dataset, Region)> = datastore + .pool_connection_for_tests() + .await? + .transaction_async(|conn| async move { + // Selecting all datasets and regions requires a full table scan + conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL).await?; + + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region::dsl; + + dsl::region + .inner_join( + dataset_dsl::dataset + .on(dsl::dataset_id.eq(dataset_dsl::id)), + ) + .select((Dataset::as_select(), Region::as_select())) + .get_results_async(&conn) + .await + }) + .await?; + + check_limit(&datasets_and_regions, limit, || { + String::from("listing datasets and regions") + }); + + datasets_and_regions + }; + + // Reconcile with the Crucible agents: are there snapshots that Nexus does + // not know about? + for (dataset, region) in datasets_and_regions { + use crucible_agent_client::types::RegionId; + use crucible_agent_client::types::State; + use crucible_agent_client::Client as CrucibleAgentClient; + + let url = format!("http://{}", dataset.address()); + let client = CrucibleAgentClient::new(&url); + + let actual_region_snapshots = client + .region_get_snapshots(&RegionId(region.id().to_string())) + .await?; + + let default = HashSet::default(); + let nexus_region_snapshots: &HashSet = + regions_to_snapshots_map.get(®ion.id()).unwrap_or(&default); + + for actual_region_snapshot in &actual_region_snapshots.snapshots { + let snapshot_id: Uuid = actual_region_snapshot.name.parse()?; + if !nexus_region_snapshots.contains(&snapshot_id) { + rows.push(Row { + dataset_id: dataset.id(), + region_id: region.id(), + snapshot_id, + dataset_addr: dataset.address(), + error: String::from( + "Nexus does not know about this snapshot!", + ), + }); + } + } + + for (_, actual_region_running_snapshot) in + &actual_region_snapshots.running_snapshots + { + let snapshot_id: Uuid = + actual_region_running_snapshot.name.parse()?; + + match actual_region_running_snapshot.state { + State::Destroyed | State::Failed | State::Tombstoned => { + // don't check, Nexus would consider this gone + } + + State::Requested | State::Created => { + if !nexus_region_snapshots.contains(&snapshot_id) { + rows.push(Row { + dataset_id: dataset.id(), + region_id: region.id(), + snapshot_id, + dataset_addr: dataset.address(), + error: String::from( + "Nexus does not know about this running snapshot!" + ), + }); + } + } + } + } + } + + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .to_string(); + + println!("{}", table); + + Ok(()) +} + fn print_name( prefix: &str, name: &str, @@ -1751,6 +2216,7 @@ fn format_record(record: &DnsRecord) -> impl Display { // Inventory async fn cmd_db_inventory( + opctx: &OpContext, datastore: &DataStore, limit: NonZeroU32, inventory_args: &InventoryArgs, @@ -1768,7 +2234,9 @@ async fn cmd_db_inventory( }) => cmd_db_inventory_collections_list(&conn, limit).await, InventoryCommands::Collections(CollectionsArgs { command: CollectionsCommands::Show(CollectionsShowArgs { id }), - }) => cmd_db_inventory_collections_show(datastore, id, limit).await, + }) => { + cmd_db_inventory_collections_show(opctx, datastore, id, limit).await + } } } @@ -1928,12 +2396,13 @@ async fn cmd_db_inventory_collections_list( } async fn cmd_db_inventory_collections_show( + opctx: &OpContext, datastore: &DataStore, id: Uuid, limit: NonZeroU32, ) -> Result<(), anyhow::Error> { let (collection, incomplete) = datastore - .inventory_collection_read_best_effort(id, limit) + .inventory_collection_read_best_effort(opctx, id, limit) .await .context("reading collection")?; if incomplete { diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index 128d4315f2..9f91d38504 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -159,6 +159,7 @@ async fn cmd_nexus_background_tasks_show( "dns_config_external", "dns_servers_external", "dns_propagation_external", + "nat_v4_garbage_collector", ] { if let Some(bgtask) = tasks.remove(name) { print_task(&bgtask); diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 7949c1eb61..fd50d80c81 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -61,6 +61,11 @@ task: "inventory_collection" collects hardware and software inventory data from the whole system +task: "nat_v4_garbage_collector" + prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a + predetermined retention policy + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT @@ -121,6 +126,11 @@ task: "inventory_collection" collects hardware and software inventory data from the whole system +task: "nat_v4_garbage_collector" + prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a + predetermined retention policy + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. @@ -168,6 +178,11 @@ task: "inventory_collection" collects hardware and software inventory data from the whole system +task: "nat_v4_garbage_collector" + prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a + predetermined retention policy + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 8162b6d9de..6bc3a85e8a 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -255,6 +255,11 @@ task: "inventory_collection" collects hardware and software inventory data from the whole system +task: "nat_v4_garbage_collector" + prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a + predetermined retention policy + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ @@ -319,6 +324,13 @@ task: "dns_propagation_external" [::1]:REDACTED_PORT success +task: "nat_v4_garbage_collector" + configured period: every 30s + currently executing: no + last completed activation: iter 2, triggered by an explicit signal + started at (s ago) and ran for ms +warning: unknown background task: "nat_v4_garbage_collector" (don't know how to interpret details: Null) + task: "external_endpoints" configured period: every 1m currently executing: no diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index e859c325a5..eaabf970a6 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -98,11 +98,13 @@ Commands: instances Print information about customer instances network Print information about the network snapshots Print information about snapshots + validate Validate the contents of the database help Print this message or the help of the given subcommand(s) Options: --db-url URL of the database SQL interface [env: OMDB_DB_URL=] - --fetch-limit limit to apply to queries that fetch rows [default: 500] + --fetch-limit limit to apply to queries that fetch rows [env: + OMDB_FETCH_LIMIT=] [default: 500] -h, --help Print help ============================================= EXECUTING COMMAND: omdb ["db", "--help"] @@ -122,11 +124,13 @@ Commands: instances Print information about customer instances network Print information about the network snapshots Print information about snapshots + validate Validate the contents of the database help Print this message or the help of the given subcommand(s) Options: --db-url URL of the database SQL interface [env: OMDB_DB_URL=] - --fetch-limit limit to apply to queries that fetch rows [default: 500] + --fetch-limit limit to apply to queries that fetch rows [env: + OMDB_FETCH_LIMIT=] [default: 500] -h, --help Print help --------------------------------------------- stderr: diff --git a/nexus/db-model/src/ipv4_nat_entry.rs b/nexus/db-model/src/ipv4_nat_entry.rs new file mode 100644 index 0000000000..570a46b5e9 --- /dev/null +++ b/nexus/db-model/src/ipv4_nat_entry.rs @@ -0,0 +1,81 @@ +use std::net::{Ipv4Addr, Ipv6Addr}; + +use super::MacAddr; +use crate::{schema::ipv4_nat_entry, Ipv4Net, Ipv6Net, SqlU16, Vni}; +use chrono::{DateTime, Utc}; +use omicron_common::api::external; +use schemars::JsonSchema; +use serde::Serialize; +use uuid::Uuid; + +/// Values used to create an Ipv4NatEntry +#[derive(Insertable, Debug, Clone)] +#[diesel(table_name = ipv4_nat_entry)] +pub struct Ipv4NatValues { + pub external_address: Ipv4Net, + pub first_port: SqlU16, + pub last_port: SqlU16, + pub sled_address: Ipv6Net, + pub vni: Vni, + pub mac: MacAddr, +} + +/// Database representation of an Ipv4 NAT Entry. +#[derive(Queryable, Debug, Clone, Selectable)] +#[diesel(table_name = ipv4_nat_entry)] +pub struct Ipv4NatEntry { + pub id: Uuid, + pub external_address: Ipv4Net, + pub first_port: SqlU16, + pub last_port: SqlU16, + pub sled_address: Ipv6Net, + pub vni: Vni, + pub mac: MacAddr, + pub version_added: i64, + pub version_removed: Option, + pub time_created: DateTime, + pub time_deleted: Option>, +} + +impl Ipv4NatEntry { + pub fn first_port(&self) -> u16 { + self.first_port.into() + } + + pub fn last_port(&self) -> u16 { + self.last_port.into() + } +} + +/// NAT Record +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct Ipv4NatEntryView { + pub external_address: Ipv4Addr, + pub first_port: u16, + pub last_port: u16, + pub sled_address: Ipv6Addr, + pub vni: external::Vni, + pub mac: external::MacAddr, + pub gen: i64, + pub deleted: bool, +} + +impl From for Ipv4NatEntryView { + fn from(value: Ipv4NatEntry) -> Self { + let (gen, deleted) = match value.version_removed { + Some(gen) => (gen, true), + None => (value.version_added, false), + }; + + Self { + external_address: value.external_address.ip(), + first_port: value.first_port(), + last_port: value.last_port(), + sled_address: value.sled_address.ip(), + vni: value.vni.0, + mac: *value.mac, + gen, + deleted, + } + } +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 7aa8a6b076..6b65eb87ec 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -53,6 +53,7 @@ mod system_update; // These actually represent subqueries, not real table. // However, they must be defined in the same crate as our tables // for join-based marker trait generation. +mod ipv4_nat_entry; pub mod queries; mod rack; mod region; @@ -124,6 +125,7 @@ pub use instance_cpu_count::*; pub use instance_state::*; pub use inventory::*; pub use ip_pool::*; +pub use ipv4_nat_entry::*; pub use ipv4net::*; pub use ipv6::*; pub use ipv6net::*; diff --git a/nexus/db-model/src/rack.rs b/nexus/db-model/src/rack.rs index 580ec155b4..f2bc7528d2 100644 --- a/nexus/db-model/src/rack.rs +++ b/nexus/db-model/src/rack.rs @@ -4,8 +4,9 @@ use crate::schema::rack; use db_macros::Asset; -use ipnetwork::IpNetwork; +use ipnetwork::{IpNetwork, Ipv6Network}; use nexus_types::{external_api::views, identity::Asset}; +use omicron_common::api; use uuid::Uuid; /// Information about a local rack. @@ -28,6 +29,22 @@ impl Rack { rack_subnet: None, } } + + pub fn subnet(&self) -> Result { + match self.rack_subnet { + Some(IpNetwork::V6(subnet)) => Ok(subnet), + Some(IpNetwork::V4(_)) => { + return Err(api::external::Error::InternalError { + internal_message: "rack subnet not IPv6".into(), + }) + } + None => { + return Err(api::external::Error::InternalError { + internal_message: "rack subnet not set".into(), + }) + } + } + } } impl From for views::Rack { diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 7c6b8bbd0a..4844f2a33f 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -489,6 +489,32 @@ table! { } } +table! { + ipv4_nat_entry (id) { + id -> Uuid, + external_address -> Inet, + first_port -> Int4, + last_port -> Int4, + sled_address -> Inet, + vni -> Int4, + mac -> Int8, + version_added -> Int8, + version_removed -> Nullable, + time_created -> Timestamptz, + time_deleted -> Nullable, + } +} + +// This is the sequence used for the version number +// in ipv4_nat_entry. +table! { + ipv4_nat_version (last_value) { + last_value -> Int8, + log_cnt -> Int8, + is_called -> Bool, + } +} + table! { external_ip (id) { id -> Uuid, @@ -1243,7 +1269,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(10, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(11, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 114b9dbe31..b743d28ee8 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -10,8 +10,6 @@ use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::queries::ALLOW_FULL_TABLE_SCAN_SQL; use crate::db::TransactionError; -use anyhow::anyhow; -use anyhow::bail; use anyhow::Context; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; @@ -44,6 +42,7 @@ use nexus_db_model::SwCaboose; use nexus_types::inventory::Collection; use omicron_common::api::external::Error; use omicron_common::api::external::InternalContext; +use omicron_common::bail_unless; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::num::NonZeroU32; @@ -798,54 +797,311 @@ impl DataStore { Ok(()) } -} -/// Extra interfaces that are not intended (and potentially unsafe) for use in -/// Nexus, but useful for testing and `omdb` -pub trait DataStoreInventoryTest: Send + Sync { - /// List all collections - /// - /// This does not paginate. - fn inventory_collections(&self) -> BoxFuture>>; + /// Attempt to read the latest collection while limiting queries to `limit` + /// records + pub async fn inventory_get_latest_collection( + &self, + opctx: &OpContext, + limit: NonZeroU32, + ) -> Result { + opctx.authorize(authz::Action::Read, &authz::INVENTORY).await?; + let conn = self.pool_connection_authorized(opctx).await?; + use db::schema::inv_collection::dsl; + let collection_id = dsl::inv_collection + .select(dsl::id) + .order_by(dsl::time_started.desc()) + .limit(1) + .first_async::(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; - /// Make a best effort to read the given collection while limiting queries - /// to `limit` results. Returns as much as it was able to get. The - /// returned bool indicates whether the returned collection might be - /// incomplete because the limit was reached. - fn inventory_collection_read_best_effort( + self.inventory_collection_read_all_or_nothing( + opctx, + collection_id, + limit, + ) + .await + } + + /// Attempt to read the given collection while limiting queries to `limit` + /// records and returning nothing if `limit` is not large enough. + async fn inventory_collection_read_all_or_nothing( &self, + opctx: &OpContext, id: Uuid, limit: NonZeroU32, - ) -> BoxFuture>; + ) -> Result { + let (collection, limit_reached) = self + .inventory_collection_read_best_effort(opctx, id, limit) + .await?; + bail_unless!( + !limit_reached, + "hit limit of {} records while loading collection", + limit + ); + Ok(collection) + } - /// Attempt to read the given collection while limiting queries to `limit` - /// records - fn inventory_collection_read_all_or_nothing( + /// Make a best effort to read the given collection while limiting queries + /// to `limit` results. Returns as much as it was able to get. The + /// returned bool indicates whether the returned collection might be + /// incomplete because the limit was reached. + pub async fn inventory_collection_read_best_effort( &self, + opctx: &OpContext, id: Uuid, limit: NonZeroU32, - ) -> BoxFuture> { - async move { - let (collection, limit_reached) = - self.inventory_collection_read_best_effort(id, limit).await?; - anyhow::ensure!( - !limit_reached, - "hit limit of {} records while loading collection", - limit + ) -> Result<(Collection, bool), Error> { + let conn = self.pool_connection_authorized(opctx).await?; + let sql_limit = i64::from(u32::from(limit)); + let usize_limit = usize::try_from(u32::from(limit)).unwrap(); + let mut limit_reached = false; + let (time_started, time_done, collector) = { + use db::schema::inv_collection::dsl; + + let collections = dsl::inv_collection + .filter(dsl::id.eq(id)) + .limit(2) + .select(InvCollection::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + bail_unless!(collections.len() == 1); + let collection = collections.into_iter().next().unwrap(); + ( + collection.time_started, + collection.time_done, + collection.collector, + ) + }; + + let errors: Vec = { + use db::schema::inv_collection_error::dsl; + dsl::inv_collection_error + .filter(dsl::inv_collection_id.eq(id)) + .order_by(dsl::idx) + .limit(sql_limit) + .select(InvCollectionError::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|e| e.message) + .collect() + }; + limit_reached = limit_reached || errors.len() == usize_limit; + + let sps: BTreeMap<_, _> = { + use db::schema::inv_service_processor::dsl; + dsl::inv_service_processor + .filter(dsl::inv_collection_id.eq(id)) + .limit(sql_limit) + .select(InvServiceProcessor::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|sp_row| { + let baseboard_id = sp_row.hw_baseboard_id; + ( + baseboard_id, + nexus_types::inventory::ServiceProcessor::from(sp_row), + ) + }) + .collect() + }; + limit_reached = limit_reached || sps.len() == usize_limit; + + let rots: BTreeMap<_, _> = { + use db::schema::inv_root_of_trust::dsl; + dsl::inv_root_of_trust + .filter(dsl::inv_collection_id.eq(id)) + .limit(sql_limit) + .select(InvRootOfTrust::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|rot_row| { + let baseboard_id = rot_row.hw_baseboard_id; + ( + baseboard_id, + nexus_types::inventory::RotState::from(rot_row), + ) + }) + .collect() + }; + limit_reached = limit_reached || rots.len() == usize_limit; + + // Collect the unique baseboard ids referenced by SPs and RoTs. + let baseboard_id_ids: BTreeSet<_> = + sps.keys().chain(rots.keys()).cloned().collect(); + // Fetch the corresponding baseboard records. + let baseboards_by_id: BTreeMap<_, _> = { + use db::schema::hw_baseboard_id::dsl; + dsl::hw_baseboard_id + .filter(dsl::id.eq_any(baseboard_id_ids)) + .limit(sql_limit) + .select(HwBaseboardId::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|bb| { + ( + bb.id, + Arc::new(nexus_types::inventory::BaseboardId::from(bb)), + ) + }) + .collect() + }; + limit_reached = limit_reached || baseboards_by_id.len() == usize_limit; + + // Having those, we can replace the keys in the maps above with + // references to the actual baseboard rather than the uuid. + let sps = sps + .into_iter() + .map(|(id, sp)| { + baseboards_by_id.get(&id).map(|bb| (bb.clone(), sp)).ok_or_else( + || { + Error::internal_error( + "missing baseboard that we should have fetched", + ) + }, + ) + }) + .collect::, _>>()?; + let rots = rots + .into_iter() + .map(|(id, rot)| { + baseboards_by_id + .get(&id) + .map(|bb| (bb.clone(), rot)) + .ok_or_else(|| { + Error::internal_error( + "missing baseboard that we should have fetched", + ) + }) + }) + .collect::, _>>()?; + + // Fetch records of cabooses found. + let inv_caboose_rows = { + use db::schema::inv_caboose::dsl; + dsl::inv_caboose + .filter(dsl::inv_collection_id.eq(id)) + .limit(sql_limit) + .select(InvCaboose::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })? + }; + limit_reached = limit_reached || inv_caboose_rows.len() == usize_limit; + + // Collect the unique sw_caboose_ids for those cabooses. + let sw_caboose_ids: BTreeSet<_> = inv_caboose_rows + .iter() + .map(|inv_caboose| inv_caboose.sw_caboose_id) + .collect(); + // Fetch the corresponing records. + let cabooses_by_id: BTreeMap<_, _> = { + use db::schema::sw_caboose::dsl; + dsl::sw_caboose + .filter(dsl::id.eq_any(sw_caboose_ids)) + .limit(sql_limit) + .select(SwCaboose::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|sw_caboose_row| { + ( + sw_caboose_row.id, + Arc::new(nexus_types::inventory::Caboose::from( + sw_caboose_row, + )), + ) + }) + .collect() + }; + limit_reached = limit_reached || cabooses_by_id.len() == usize_limit; + + // Assemble the lists of cabooses found. + let mut cabooses_found = BTreeMap::new(); + for c in inv_caboose_rows { + let by_baseboard = cabooses_found + .entry(nexus_types::inventory::CabooseWhich::from(c.which)) + .or_insert_with(BTreeMap::new); + let Some(bb) = baseboards_by_id.get(&c.hw_baseboard_id) else { + let msg = format!( + "unknown baseboard found in inv_caboose: {}", + c.hw_baseboard_id + ); + return Err(Error::internal_error(&msg)); + }; + let Some(sw_caboose) = cabooses_by_id.get(&c.sw_caboose_id) else { + let msg = format!( + "unknown caboose found in inv_caboose: {}", + c.sw_caboose_id + ); + return Err(Error::internal_error(&msg)); + }; + + let previous = by_baseboard.insert( + bb.clone(), + nexus_types::inventory::CabooseFound { + time_collected: c.time_collected, + source: c.source, + caboose: sw_caboose.clone(), + }, + ); + bail_unless!( + previous.is_none(), + "duplicate caboose found: {:?} baseboard {:?}", + c.which, + c.hw_baseboard_id ); - Ok(collection) } - .boxed() + + Ok(( + Collection { + id, + errors, + time_started, + time_done, + collector, + baseboards: baseboards_by_id.values().cloned().collect(), + cabooses: cabooses_by_id.values().cloned().collect(), + sps, + rots, + cabooses_found, + }, + limit_reached, + )) } } +/// Extra interfaces that are not intended (and potentially unsafe) for use in +/// Nexus, but useful for testing and `omdb` +pub trait DataStoreInventoryTest: Send + Sync { + /// List all collections + /// + /// This does not paginate. + fn inventory_collections(&self) -> BoxFuture>>; +} + impl DataStoreInventoryTest for DataStore { fn inventory_collections(&self) -> BoxFuture>> { async { let conn = self .pool_connection_for_tests() .await - .context("getting connectoin")?; + .context("getting connection")?; conn.transaction_async(|conn| async move { conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL) .await @@ -863,257 +1119,11 @@ impl DataStoreInventoryTest for DataStore { } .boxed() } - - // This function could move into the datastore if it proves helpful. We'd - // need to work out how to report the usual type of Error. For now we don't - // need it so we limit its scope to the test suite. - fn inventory_collection_read_best_effort( - &self, - id: Uuid, - limit: NonZeroU32, - ) -> BoxFuture> { - async move { - let conn = &self - .pool_connection_for_tests() - .await - .context("getting connection")?; - let sql_limit = i64::from(u32::from(limit)); - let usize_limit = usize::try_from(u32::from(limit)).unwrap(); - let mut limit_reached = false; - let (time_started, time_done, collector) = { - use db::schema::inv_collection::dsl; - - let collections = dsl::inv_collection - .filter(dsl::id.eq(id)) - .limit(2) - .select(InvCollection::as_select()) - .load_async(&**conn) - .await - .context("loading collection")?; - anyhow::ensure!(collections.len() == 1); - let collection = collections.into_iter().next().unwrap(); - ( - collection.time_started, - collection.time_done, - collection.collector, - ) - }; - - let errors: Vec = { - use db::schema::inv_collection_error::dsl; - dsl::inv_collection_error - .filter(dsl::inv_collection_id.eq(id)) - .order_by(dsl::idx) - .limit(sql_limit) - .select(InvCollectionError::as_select()) - .load_async(&**conn) - .await - .context("loading collection errors")? - .into_iter() - .map(|e| e.message) - .collect() - }; - limit_reached = limit_reached || errors.len() == usize_limit; - - let sps: BTreeMap<_, _> = { - use db::schema::inv_service_processor::dsl; - dsl::inv_service_processor - .filter(dsl::inv_collection_id.eq(id)) - .limit(sql_limit) - .select(InvServiceProcessor::as_select()) - .load_async(&**conn) - .await - .context("loading service processors")? - .into_iter() - .map(|sp_row| { - let baseboard_id = sp_row.hw_baseboard_id; - ( - baseboard_id, - nexus_types::inventory::ServiceProcessor::from( - sp_row, - ), - ) - }) - .collect() - }; - limit_reached = limit_reached || sps.len() == usize_limit; - - let rots: BTreeMap<_, _> = { - use db::schema::inv_root_of_trust::dsl; - dsl::inv_root_of_trust - .filter(dsl::inv_collection_id.eq(id)) - .limit(sql_limit) - .select(InvRootOfTrust::as_select()) - .load_async(&**conn) - .await - .context("loading roots of trust")? - .into_iter() - .map(|rot_row| { - let baseboard_id = rot_row.hw_baseboard_id; - ( - baseboard_id, - nexus_types::inventory::RotState::from(rot_row), - ) - }) - .collect() - }; - limit_reached = limit_reached || rots.len() == usize_limit; - - // Collect the unique baseboard ids referenced by SPs and RoTs. - let baseboard_id_ids: BTreeSet<_> = - sps.keys().chain(rots.keys()).cloned().collect(); - // Fetch the corresponding baseboard records. - let baseboards_by_id: BTreeMap<_, _> = { - use db::schema::hw_baseboard_id::dsl; - dsl::hw_baseboard_id - .filter(dsl::id.eq_any(baseboard_id_ids)) - .limit(sql_limit) - .select(HwBaseboardId::as_select()) - .load_async(&**conn) - .await - .context("loading baseboards")? - .into_iter() - .map(|bb| { - ( - bb.id, - Arc::new( - nexus_types::inventory::BaseboardId::from(bb), - ), - ) - }) - .collect() - }; - limit_reached = - limit_reached || baseboards_by_id.len() == usize_limit; - - // Having those, we can replace the keys in the maps above with - // references to the actual baseboard rather than the uuid. - let sps = sps - .into_iter() - .map(|(id, sp)| { - baseboards_by_id - .get(&id) - .map(|bb| (bb.clone(), sp)) - .ok_or_else(|| { - anyhow!( - "missing baseboard that we should have fetched" - ) - }) - }) - .collect::, _>>()?; - let rots = - rots.into_iter() - .map(|(id, rot)| { - baseboards_by_id - .get(&id) - .map(|bb| (bb.clone(), rot)) - .ok_or_else(|| { - anyhow!("missing baseboard that we should have fetched") - }) - }) - .collect::, _>>()?; - - // Fetch records of cabooses found. - let inv_caboose_rows = { - use db::schema::inv_caboose::dsl; - dsl::inv_caboose - .filter(dsl::inv_collection_id.eq(id)) - .limit(sql_limit) - .select(InvCaboose::as_select()) - .load_async(&**conn) - .await - .context("loading inv_cabooses")? - }; - limit_reached = - limit_reached || inv_caboose_rows.len() == usize_limit; - - // Collect the unique sw_caboose_ids for those cabooses. - let sw_caboose_ids: BTreeSet<_> = inv_caboose_rows - .iter() - .map(|inv_caboose| inv_caboose.sw_caboose_id) - .collect(); - // Fetch the corresponing records. - let cabooses_by_id: BTreeMap<_, _> = { - use db::schema::sw_caboose::dsl; - dsl::sw_caboose - .filter(dsl::id.eq_any(sw_caboose_ids)) - .limit(sql_limit) - .select(SwCaboose::as_select()) - .load_async(&**conn) - .await - .context("loading sw_cabooses")? - .into_iter() - .map(|sw_caboose_row| { - ( - sw_caboose_row.id, - Arc::new(nexus_types::inventory::Caboose::from( - sw_caboose_row, - )), - ) - }) - .collect() - }; - limit_reached = - limit_reached || cabooses_by_id.len() == usize_limit; - - // Assemble the lists of cabooses found. - let mut cabooses_found = BTreeMap::new(); - for c in inv_caboose_rows { - let by_baseboard = cabooses_found - .entry(nexus_types::inventory::CabooseWhich::from(c.which)) - .or_insert_with(BTreeMap::new); - let Some(bb) = baseboards_by_id.get(&c.hw_baseboard_id) else { - bail!( - "unknown baseboard found in inv_caboose: {}", - c.hw_baseboard_id - ); - }; - let Some(sw_caboose) = cabooses_by_id.get(&c.sw_caboose_id) - else { - bail!( - "unknown caboose found in inv_caboose: {}", - c.sw_caboose_id - ); - }; - - let previous = by_baseboard.insert( - bb.clone(), - nexus_types::inventory::CabooseFound { - time_collected: c.time_collected, - source: c.source, - caboose: sw_caboose.clone(), - }, - ); - anyhow::ensure!( - previous.is_none(), - "duplicate caboose found: {:?} baseboard {:?}", - c.which, - c.hw_baseboard_id - ); - } - - Ok(( - Collection { - id, - errors, - time_started, - time_done, - collector, - baseboards: baseboards_by_id.values().cloned().collect(), - cabooses: cabooses_by_id.values().cloned().collect(), - sps, - rots, - cabooses_found, - }, - limit_reached, - )) - } - .boxed() - } } #[cfg(test)] mod test { + use crate::context::OpContext; use crate::db::datastore::datastore_test; use crate::db::datastore::inventory::DataStoreInventoryTest; use crate::db::datastore::DataStore; @@ -1136,11 +1146,14 @@ mod test { use uuid::Uuid; async fn read_collection( + opctx: &OpContext, datastore: &DataStore, id: Uuid, ) -> anyhow::Result { let limit = NonZeroU32::new(1000).unwrap(); - datastore.inventory_collection_read_all_or_nothing(id, limit).await + Ok(datastore + .inventory_collection_read_all_or_nothing(opctx, id, limit) + .await?) } async fn count_baseboards_cabooses( @@ -1186,9 +1199,10 @@ mod test { // Read it back. let conn = datastore.pool_connection_for_tests().await.unwrap(); - let collection_read = read_collection(&datastore, collection1.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection1.id) + .await + .expect("failed to read collection back"); assert_eq!(collection1, collection_read); // There ought to be no baseboards or cabooses in the databases from @@ -1208,9 +1222,10 @@ mod test { .inventory_insert_collection(&opctx, &collection2) .await .expect("failed to insert collection"); - let collection_read = read_collection(&datastore, collection2.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection2.id) + .await + .expect("failed to read collection back"); assert_eq!(collection2, collection_read); // Verify that we have exactly the set of cabooses and baseboards in the // databases that came from this first non-empty collection. @@ -1221,6 +1236,18 @@ mod test { assert_eq!(collection2.baseboards.len(), nbaseboards); assert_eq!(collection2.cabooses.len(), ncabooses); + // Check that we get an error on the limit being reached for + // `read_all_or_nothing` + let limit = NonZeroU32::new(1).unwrap(); + assert!(datastore + .inventory_collection_read_all_or_nothing( + &opctx, + collection2.id, + limit + ) + .await + .is_err()); + // Now insert an equivalent collection again. Verify the distinct // baseboards and cabooses again. This is important: the insertion // process should re-use the baseboards and cabooses from the previous @@ -1231,9 +1258,10 @@ mod test { .inventory_insert_collection(&opctx, &collection3) .await .expect("failed to insert collection"); - let collection_read = read_collection(&datastore, collection3.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection3.id) + .await + .expect("failed to read collection back"); assert_eq!(collection3, collection_read); // Verify that we have the same number of cabooses and baseboards, since // those didn't change. @@ -1275,9 +1303,10 @@ mod test { .inventory_insert_collection(&opctx, &collection4) .await .expect("failed to insert collection"); - let collection_read = read_collection(&datastore, collection4.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection4.id) + .await + .expect("failed to read collection back"); assert_eq!(collection4, collection_read); // Verify the number of baseboards and collections again. assert_eq!( @@ -1302,9 +1331,10 @@ mod test { .inventory_insert_collection(&opctx, &collection5) .await .expect("failed to insert collection"); - let collection_read = read_collection(&datastore, collection5.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection5.id) + .await + .expect("failed to read collection back"); assert_eq!(collection5, collection_read); assert_eq!(collection5.baseboards.len(), collection3.baseboards.len()); assert_eq!(collection5.cabooses.len(), collection3.cabooses.len()); @@ -1433,19 +1463,19 @@ mod test { ); // If we try to fetch a pruned collection, we should get nothing. - let _ = read_collection(&datastore, collection4.id) + let _ = read_collection(&opctx, &datastore, collection4.id) .await .expect_err("unexpectedly read pruned collection"); // But we should still be able to fetch the collections that do exist. let collection_read = - read_collection(&datastore, collection5.id).await.unwrap(); + read_collection(&opctx, &datastore, collection5.id).await.unwrap(); assert_eq!(collection5, collection_read); let collection_read = - read_collection(&datastore, collection6.id).await.unwrap(); + read_collection(&opctx, &datastore, collection6.id).await.unwrap(); assert_eq!(collection6, collection_read); let collection_read = - read_collection(&datastore, collection7.id).await.unwrap(); + read_collection(&opctx, &datastore, collection7.id).await.unwrap(); assert_eq!(collection7, collection_read); // We should prune more than one collection, if needed. We'll wind up diff --git a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs new file mode 100644 index 0000000000..274937b299 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs @@ -0,0 +1,440 @@ +use super::DataStore; +use crate::context::OpContext; +use crate::db; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::model::{Ipv4NatEntry, Ipv4NatValues}; +use async_bb8_diesel::AsyncRunQueryDsl; +use chrono::{DateTime, Utc}; +use diesel::prelude::*; +use diesel::sql_types::BigInt; +use nexus_db_model::ExternalIp; +use nexus_db_model::Ipv4NatEntryView; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::LookupResult; +use omicron_common::api::external::LookupType; +use omicron_common::api::external::ResourceType; + +impl DataStore { + pub async fn ensure_ipv4_nat_entry( + &self, + opctx: &OpContext, + nat_entry: Ipv4NatValues, + ) -> CreateResult<()> { + use db::schema::ipv4_nat_entry::dsl; + use diesel::sql_types; + + // Look up any NAT entries that already have the exact parameters + // we're trying to INSERT. + let matching_entry_subquery = dsl::ipv4_nat_entry + .filter(dsl::external_address.eq(nat_entry.external_address)) + .filter(dsl::first_port.eq(nat_entry.first_port)) + .filter(dsl::last_port.eq(nat_entry.last_port)) + .filter(dsl::sled_address.eq(nat_entry.sled_address)) + .filter(dsl::vni.eq(nat_entry.vni)) + .filter(dsl::mac.eq(nat_entry.mac)) + .select(( + dsl::external_address, + dsl::first_port, + dsl::last_port, + dsl::sled_address, + dsl::vni, + dsl::mac, + )); + + // SELECT exactly the values we're trying to INSERT, but only + // if it does not already exist. + let new_entry_subquery = diesel::dsl::select(( + nat_entry.external_address.into_sql::(), + nat_entry.first_port.into_sql::(), + nat_entry.last_port.into_sql::(), + nat_entry.sled_address.into_sql::(), + nat_entry.vni.into_sql::(), + nat_entry.mac.into_sql::(), + )) + .filter(diesel::dsl::not(diesel::dsl::exists(matching_entry_subquery))); + + diesel::insert_into(dsl::ipv4_nat_entry) + .values(new_entry_subquery) + .into_columns(( + dsl::external_address, + dsl::first_port, + dsl::last_port, + dsl::sled_address, + dsl::vni, + dsl::mac, + )) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } + + pub async fn ipv4_nat_delete( + &self, + opctx: &OpContext, + nat_entry: &Ipv4NatEntry, + ) -> DeleteResult { + use db::schema::ipv4_nat_entry::dsl; + + let updated_rows = diesel::update(dsl::ipv4_nat_entry) + .set(( + dsl::version_removed.eq(ipv4_nat_next_version().nullable()), + dsl::time_deleted.eq(Utc::now()), + )) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::version_removed.is_null()) + .filter(dsl::id.eq(nat_entry.id)) + .filter(dsl::version_added.eq(nat_entry.version_added)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + if updated_rows == 0 { + return Err(Error::ObjectNotFound { + type_name: ResourceType::Ipv4NatEntry, + lookup_type: LookupType::ByCompositeId( + "id, version_added".to_string(), + ), + }); + } + Ok(()) + } + + pub async fn ipv4_nat_find_by_id( + &self, + opctx: &OpContext, + id: uuid::Uuid, + ) -> LookupResult { + use db::schema::ipv4_nat_entry::dsl; + + let result = dsl::ipv4_nat_entry + .filter(dsl::id.eq(id)) + .select(Ipv4NatEntry::as_select()) + .limit(1) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + if let Some(nat_entry) = result.first() { + Ok(nat_entry.clone()) + } else { + Err(Error::InvalidRequest { + message: "no matching records".to_string(), + }) + } + } + + pub async fn ipv4_nat_delete_by_external_ip( + &self, + opctx: &OpContext, + external_ip: &ExternalIp, + ) -> DeleteResult { + use db::schema::ipv4_nat_entry::dsl; + + let updated_rows = diesel::update(dsl::ipv4_nat_entry) + .set(( + dsl::version_removed.eq(ipv4_nat_next_version().nullable()), + dsl::time_deleted.eq(Utc::now()), + )) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::version_removed.is_null()) + .filter(dsl::external_address.eq(external_ip.ip)) + .filter(dsl::first_port.eq(external_ip.first_port)) + .filter(dsl::last_port.eq(external_ip.last_port)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + if updated_rows == 0 { + return Err(Error::ObjectNotFound { + type_name: ResourceType::Ipv4NatEntry, + lookup_type: LookupType::ByCompositeId( + "external_ip, first_port, last_port".to_string(), + ), + }); + } + Ok(()) + } + + pub async fn ipv4_nat_find_by_values( + &self, + opctx: &OpContext, + values: Ipv4NatValues, + ) -> LookupResult { + use db::schema::ipv4_nat_entry::dsl; + let result = dsl::ipv4_nat_entry + .filter(dsl::external_address.eq(values.external_address)) + .filter(dsl::first_port.eq(values.first_port)) + .filter(dsl::last_port.eq(values.last_port)) + .filter(dsl::mac.eq(values.mac)) + .filter(dsl::sled_address.eq(values.sled_address)) + .filter(dsl::vni.eq(values.vni)) + .filter(dsl::time_deleted.is_null()) + .select(Ipv4NatEntry::as_select()) + .limit(1) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + if let Some(nat_entry) = result.first() { + Ok(nat_entry.clone()) + } else { + Err(Error::InvalidRequest { + message: "no matching records".to_string(), + }) + } + } + + pub async fn ipv4_nat_list_since_version( + &self, + opctx: &OpContext, + version: i64, + limit: u32, + ) -> ListResultVec { + use db::schema::ipv4_nat_entry::dsl; + + let list = dsl::ipv4_nat_entry + .filter( + dsl::version_added + .gt(version) + .or(dsl::version_removed.gt(version)), + ) + .limit(limit as i64) + .select(Ipv4NatEntry::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(list) + } + + pub async fn ipv4_nat_changeset( + &self, + opctx: &OpContext, + version: i64, + limit: u32, + ) -> ListResultVec { + let nat_entries = + self.ipv4_nat_list_since_version(opctx, version, limit).await?; + let nat_entries: Vec = + nat_entries.iter().map(|e| e.clone().into()).collect(); + Ok(nat_entries) + } + + pub async fn ipv4_nat_current_version( + &self, + opctx: &OpContext, + ) -> LookupResult { + use db::schema::ipv4_nat_version::dsl; + + let latest: Option = dsl::ipv4_nat_version + .select(diesel::dsl::max(dsl::last_value)) + .first_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + match latest { + Some(value) => Ok(value), + None => Err(Error::InvalidRequest { + message: "sequence table is empty!".to_string(), + }), + } + } + + pub async fn ipv4_nat_cleanup( + &self, + opctx: &OpContext, + version: i64, + before_timestamp: DateTime, + ) -> DeleteResult { + use db::schema::ipv4_nat_entry::dsl; + + diesel::delete(dsl::ipv4_nat_entry) + .filter(dsl::version_removed.lt(version)) + .filter(dsl::time_deleted.lt(before_timestamp)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } +} + +fn ipv4_nat_next_version() -> diesel::expression::SqlLiteral { + diesel::dsl::sql::("nextval('omicron.public.ipv4_nat_version')") +} + +#[cfg(test)] +mod test { + use std::str::FromStr; + + use crate::db::datastore::datastore_test; + use chrono::Utc; + use nexus_db_model::{Ipv4NatValues, MacAddr, Vni}; + use nexus_test_utils::db::test_setup_database; + use omicron_common::api::external; + use omicron_test_utils::dev; + + // Test our ability to track additions and deletions since a given version number + #[tokio::test] + async fn nat_version_tracking() { + let logctx = dev::test_setup_log("test_nat_version_tracking"); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // We should not have any NAT entries at this moment + let initial_state = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert!(initial_state.is_empty()); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 0 + ); + + // Each change (creation / deletion) to the NAT table should increment the + // version number of the row in the NAT table + let external_address = external::Ipv4Net( + ipnetwork::Ipv4Network::try_from("10.0.0.100").unwrap(), + ); + + let sled_address = external::Ipv6Net( + ipnetwork::Ipv6Network::try_from("fd00:1122:3344:104::1").unwrap(), + ); + + // Add a nat entry. + let nat1 = Ipv4NatValues { + external_address: external_address.into(), + first_port: 0.into(), + last_port: 999.into(), + sled_address: sled_address.into(), + vni: Vni(external::Vni::random()), + mac: MacAddr( + external::MacAddr::from_str("A8:40:25:F5:EB:2A").unwrap(), + ), + }; + + datastore.ensure_ipv4_nat_entry(&opctx, nat1.clone()).await.unwrap(); + let first_entry = + datastore.ipv4_nat_find_by_values(&opctx, nat1).await.unwrap(); + + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + // The NAT table has undergone one change. One entry has been added, + // none deleted, so we should be at version 1. + assert_eq!(nat_entries.len(), 1); + assert_eq!(nat_entries.last().unwrap().version_added, 1); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 1 + ); + + // Add another nat entry. + let nat2 = Ipv4NatValues { + external_address: external_address.into(), + first_port: 1000.into(), + last_port: 1999.into(), + sled_address: sled_address.into(), + vni: Vni(external::Vni::random()), + mac: MacAddr( + external::MacAddr::from_str("A8:40:25:F5:EB:2B").unwrap(), + ), + }; + + datastore.ensure_ipv4_nat_entry(&opctx, nat2).await.unwrap(); + + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + // The NAT table has undergone two changes. Two entries have been + // added, none deleted, so we should be at version 2. + let nat_entry = + nat_entries.iter().find(|e| e.version_added == 2).unwrap(); + assert_eq!(nat_entries.len(), 2); + assert_eq!(nat_entry.version_added, 2); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 2 + ); + + // Test Cleanup logic + // Cleanup should only perma-delete entries that are older than a + // specified version number and whose `time_deleted` field is + // older than a specified age. + let time_cutoff = Utc::now(); + datastore.ipv4_nat_cleanup(&opctx, 2, time_cutoff).await.unwrap(); + + // Nothing should have changed (no records currently marked for deletion) + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert_eq!(nat_entries.len(), 2); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 2 + ); + + // Delete the first nat entry. It should show up as a later version number. + datastore.ipv4_nat_delete(&opctx, &first_entry).await.unwrap(); + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + // The NAT table has undergone three changes. Two entries have been + // added, one deleted, so we should be at version 3. Since the + // first entry was marked for deletion (and it was the third change), + // the first entry's version number should now be 3. + let nat_entry = + nat_entries.iter().find(|e| e.version_removed.is_some()).unwrap(); + assert_eq!(nat_entries.len(), 2); + assert_eq!(nat_entry.version_removed, Some(3)); + assert_eq!(nat_entry.id, first_entry.id); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 3 + ); + + // Try cleaning up with the old version and time cutoff values + datastore.ipv4_nat_cleanup(&opctx, 2, time_cutoff).await.unwrap(); + + // Try cleaning up with a greater version and old time cutoff values + datastore.ipv4_nat_cleanup(&opctx, 6, time_cutoff).await.unwrap(); + + // Try cleaning up with a older version and newer time cutoff values + datastore.ipv4_nat_cleanup(&opctx, 2, Utc::now()).await.unwrap(); + + // Both records should still exist (soft deleted record is newer than cutoff + // values ) + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert_eq!(nat_entries.len(), 2); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 3 + ); + + // Try cleaning up with a both cutoff values increased + datastore.ipv4_nat_cleanup(&opctx, 4, Utc::now()).await.unwrap(); + + // Soft deleted NAT entry should be removed from the table + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert_eq!(nat_entries.len(), 1); + + // version should be unchanged + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 3 + ); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 91373f6875..8be3386183 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -63,6 +63,7 @@ mod image; mod instance; mod inventory; mod ip_pool; +mod ipv4_nat_entry; mod network_interface; mod oximeter; mod physical_disk; @@ -102,6 +103,7 @@ pub use rack::RackInit; pub use silo::Discoverability; pub use switch_port::SwitchPortSettingsCombinedResult; pub use virtual_provisioning_collection::StorageType; +pub use volume::read_only_resources_associated_with_volume; pub use volume::CrucibleResources; pub use volume::CrucibleTargets; diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index 5d753f0742..1e64d784f7 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -1024,7 +1024,7 @@ impl DataStore { /// Return the targets from a VolumeConstructionRequest. /// /// The targets of a volume construction request map to resources. -fn read_only_resources_associated_with_volume( +pub fn read_only_resources_associated_with_volume( vcr: &VolumeConstructionRequest, crucible_targets: &mut CrucibleTargets, ) { diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index efc9aa9c27..3679fa8196 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -92,6 +92,7 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 +nat_cleanup.period_secs = 30 # How frequently to collect hardware/software inventory from the whole system # (even if we don't have reason to believe anything has changed). inventory.period_secs = 600 diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index b000dd9bda..d27248ffdc 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -10,12 +10,15 @@ use super::dns_propagation; use super::dns_servers; use super::external_endpoints; use super::inventory_collection; +use super::nat_cleanup; use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; +use omicron_common::api::internal::shared::SwitchLocation; use omicron_common::nexus_config::BackgroundTaskConfig; use omicron_common::nexus_config::DnsTasksConfig; use std::collections::BTreeMap; +use std::collections::HashMap; use std::sync::Arc; use uuid::Uuid; @@ -44,6 +47,8 @@ pub struct BackgroundTasks { pub external_endpoints: tokio::sync::watch::Receiver< Option, >, + /// task handle for the ipv4 nat entry garbage collector + pub nat_cleanup: common::TaskHandle, /// task handle for the task that collects inventory pub task_inventory_collection: common::TaskHandle, @@ -55,6 +60,7 @@ impl BackgroundTasks { opctx: &OpContext, datastore: Arc, config: &BackgroundTaskConfig, + dpd_clients: &HashMap>, nexus_id: Uuid, resolver: internal_dns::resolver::Resolver, ) -> BackgroundTasks { @@ -96,6 +102,23 @@ impl BackgroundTasks { (task, watcher_channel) }; + let nat_cleanup = { + driver.register( + "nat_v4_garbage_collector".to_string(), + String::from( + "prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table \ + based on a predetermined retention policy", + ), + config.nat_cleanup.period_secs, + Box::new(nat_cleanup::Ipv4NatGarbageCollector::new( + datastore.clone(), + dpd_clients.values().map(|client| client.clone()).collect(), + )), + opctx.child(BTreeMap::new()), + vec![], + ) + }; + // Background task: inventory collector let task_inventory_collection = { let collector = inventory_collection::InventoryCollector::new( @@ -128,6 +151,7 @@ impl BackgroundTasks { task_external_dns_servers, task_external_endpoints, external_endpoints, + nat_cleanup, task_inventory_collection, } } diff --git a/nexus/src/app/background/mod.rs b/nexus/src/app/background/mod.rs index e1f474b41a..954207cb3c 100644 --- a/nexus/src/app/background/mod.rs +++ b/nexus/src/app/background/mod.rs @@ -11,6 +11,7 @@ mod dns_servers; mod external_endpoints; mod init; mod inventory_collection; +mod nat_cleanup; mod status; pub use common::Driver; diff --git a/nexus/src/app/background/nat_cleanup.rs b/nexus/src/app/background/nat_cleanup.rs new file mode 100644 index 0000000000..1691d96a4b --- /dev/null +++ b/nexus/src/app/background/nat_cleanup.rs @@ -0,0 +1,111 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Background task for garbage collecting ipv4_nat_entry table. +//! Responsible for cleaning up soft deleted entries once they +//! have been propagated to running dpd instances. + +use super::common::BackgroundTask; +use chrono::{Duration, Utc}; +use futures::future::BoxFuture; +use futures::FutureExt; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use serde_json::json; +use std::sync::Arc; + +/// Background task that periodically prunes soft-deleted entries +/// from ipv4_nat_entry table +pub struct Ipv4NatGarbageCollector { + datastore: Arc, + dpd_clients: Vec>, +} + +impl Ipv4NatGarbageCollector { + pub fn new( + datastore: Arc, + dpd_clients: Vec>, + ) -> Ipv4NatGarbageCollector { + Ipv4NatGarbageCollector { datastore, dpd_clients } + } +} + +impl BackgroundTask for Ipv4NatGarbageCollector { + fn activate<'a, 'b, 'c>( + &'a mut self, + opctx: &'b OpContext, + ) -> BoxFuture<'c, serde_json::Value> + where + 'a: 'c, + 'b: 'c, + { + async { + let log = &opctx.log; + + let result = self.datastore.ipv4_nat_current_version(opctx).await; + + let mut min_gen = match result { + Ok(gen) => gen, + Err(error) => { + warn!( + &log, + "failed to read generation of database"; + "error" => format!("{:#}", error) + ); + return json!({ + "error": + format!( + "failed to read generation of database: \ + {:#}", + error + ) + }); + } + }; + + for client in &self.dpd_clients { + let response = client.ipv4_nat_generation().await; + match response { + Ok(gen) => min_gen = std::cmp::min(min_gen, *gen), + Err(error) => { + warn!( + &log, + "failed to read generation of dpd"; + "error" => format!("{:#}", error) + ); + return json!({ + "error": + format!( + "failed to read generation of dpd: \ + {:#}", + error + ) + }); + } + } + } + + let retention_threshold = Utc::now() - Duration::weeks(2); + + let result = self + .datastore + .ipv4_nat_cleanup(opctx, min_gen, retention_threshold) + .await + .unwrap(); + + let rv = serde_json::to_value(&result).unwrap_or_else(|error| { + json!({ + "error": + format!( + "failed to serialize final value: {:#}", + error + ) + }) + }); + + rv + } + .boxed() + } +} diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs index 0f52cbd260..abb8c744e1 100644 --- a/nexus/src/app/instance_network.rs +++ b/nexus/src/app/instance_network.rs @@ -5,6 +5,10 @@ //! Routines that manage instance-related networking state. use crate::app::sagas::retry_until_known_result; +use ipnetwork::IpNetwork; +use ipnetwork::Ipv6Network; +use nexus_db_model::Ipv4NatValues; +use nexus_db_model::Vni as DbVni; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -12,6 +16,8 @@ use nexus_db_queries::db::identity::Asset; use nexus_db_queries::db::lookup::LookupPath; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; +use omicron_common::api::external::Ipv4Net; +use omicron_common::api::external::Ipv6Net; use omicron_common::api::internal::nexus; use omicron_common::api::internal::shared::SwitchLocation; use sled_agent_client::types::DeleteVirtualNetworkInterfaceHost; @@ -330,8 +336,6 @@ impl super::Nexus { )) })?; - let vni: u32 = network_interface.vni.into(); - info!(log, "looking up instance's external IPs"; "instance_id" => %instance_id); @@ -349,6 +353,9 @@ impl super::Nexus { } } + let sled_address = + Ipv6Net(Ipv6Network::new(*sled_ip_address.ip(), 128).unwrap()); + for target_ip in ips .iter() .enumerate() @@ -361,29 +368,58 @@ impl super::Nexus { }) .map(|(_, ip)| ip) { - retry_until_known_result(log, || async { - dpd_client - .ensure_nat_entry( - &log, - target_ip.ip, - dpd_client::types::MacAddr { - a: mac_address.into_array(), - }, - *target_ip.first_port, - *target_ip.last_port, - vni, - sled_ip_address.ip(), - ) - .await - }) - .await - .map_err(|e| { - Error::internal_error(&format!( - "failed to ensure dpd entry: {e}" - )) - })?; + // For each external ip, add a nat entry to the database + self.ensure_nat_entry( + target_ip, + sled_address, + &network_interface, + mac_address, + opctx, + ) + .await?; } + // Notify dendrite that there are changes for it to reconcile. + // In the event of a failure to notify dendrite, we'll log an error + // and rely on dendrite's RPW timer to catch it up. + if let Err(e) = dpd_client.ipv4_nat_trigger_update().await { + error!(self.log, "failed to notify dendrite of nat updates"; "error" => ?e); + }; + + Ok(()) + } + + async fn ensure_nat_entry( + &self, + target_ip: &nexus_db_model::ExternalIp, + sled_address: Ipv6Net, + network_interface: &sled_agent_client::types::NetworkInterface, + mac_address: macaddr::MacAddr6, + opctx: &OpContext, + ) -> Result<(), Error> { + match target_ip.ip { + IpNetwork::V4(v4net) => { + let nat_entry = Ipv4NatValues { + external_address: Ipv4Net(v4net).into(), + first_port: target_ip.first_port, + last_port: target_ip.last_port, + sled_address: sled_address.into(), + vni: DbVni(network_interface.vni.clone().into()), + mac: nexus_db_model::MacAddr( + omicron_common::api::external::MacAddr(mac_address), + ), + }; + self.db_datastore + .ensure_ipv4_nat_entry(opctx, nat_entry) + .await?; + } + IpNetwork::V6(_v6net) => { + // TODO: implement handling of v6 nat. + return Err(Error::InternalError { + internal_message: "ipv6 nat is not yet implemented".into(), + }); + } + }; Ok(()) } @@ -419,55 +455,54 @@ impl super::Nexus { let mut errors = vec![]; for entry in external_ips { - for switch in &boundary_switches { - debug!(log, "deleting instance nat mapping"; - "instance_id" => %instance_id, - "switch" => switch.to_string(), - "entry" => #?entry); - - let client_result = - self.dpd_clients.get(switch).ok_or_else(|| { - Error::internal_error(&format!( - "unable to find dendrite client for {switch}" - )) - }); - - let dpd_client = match client_result { - Ok(client) => client, - Err(new_error) => { - errors.push(new_error); - continue; + // Soft delete the NAT entry + match self + .db_datastore + .ipv4_nat_delete_by_external_ip(&opctx, &entry) + .await + { + Ok(_) => Ok(()), + Err(err) => match err { + Error::ObjectNotFound { .. } => { + warn!(log, "no matching nat entries to soft delete"); + Ok(()) } - }; + _ => { + let message = format!( + "failed to delete nat entry due to error: {err:?}" + ); + error!(log, "{}", message); + Err(Error::internal_error(&message)) + } + }, + }?; + } - let result = retry_until_known_result(log, || async { - dpd_client - .ensure_nat_entry_deleted( - log, - entry.ip, - *entry.first_port, - ) - .await - }) - .await; - - if let Err(e) = result { - let e = Error::internal_error(&format!( - "failed to delete nat entry via dpd: {e}" - )); - - error!(log, "error deleting nat mapping: {e:#?}"; - "instance_id" => %instance_id, - "switch" => switch.to_string(), - "entry" => #?entry); - errors.push(e); - } else { - debug!(log, "deleting nat mapping successful"; - "instance_id" => %instance_id, - "switch" => switch.to_string(), - "entry" => #?entry); + for switch in &boundary_switches { + debug!(&self.log, "notifying dendrite of updates"; + "instance_id" => %authz_instance.id(), + "switch" => switch.to_string()); + + let client_result = self.dpd_clients.get(switch).ok_or_else(|| { + Error::internal_error(&format!( + "unable to find dendrite client for {switch}" + )) + }); + + let dpd_client = match client_result { + Ok(client) => client, + Err(new_error) => { + errors.push(new_error); + continue; } - } + }; + + // Notify dendrite that there are changes for it to reconcile. + // In the event of a failure to notify dendrite, we'll log an error + // and rely on dendrite's RPW timer to catch it up. + if let Err(e) = dpd_client.ipv4_nat_trigger_update().await { + error!(self.log, "failed to notify dendrite of nat updates"; "error" => ?e); + }; } if let Some(e) = errors.into_iter().nth(0) { @@ -496,32 +531,48 @@ impl super::Nexus { let boundary_switches = self.boundary_switches(opctx).await?; for external_ip in external_ips { - for switch in &boundary_switches { - debug!(&self.log, "deleting instance nat mapping"; + match self + .db_datastore + .ipv4_nat_delete_by_external_ip(&opctx, &external_ip) + .await + { + Ok(_) => Ok(()), + Err(err) => match err { + Error::ObjectNotFound { .. } => { + warn!( + self.log, + "no matching nat entries to soft delete" + ); + Ok(()) + } + _ => { + let message = format!( + "failed to delete nat entry due to error: {err:?}" + ); + error!(self.log, "{}", message); + Err(Error::internal_error(&message)) + } + }, + }?; + } + + for switch in &boundary_switches { + debug!(&self.log, "notifying dendrite of updates"; "instance_id" => %authz_instance.id(), - "switch" => switch.to_string(), - "entry" => #?external_ip); - - let dpd_client = - self.dpd_clients.get(switch).ok_or_else(|| { - Error::internal_error(&format!( - "unable to find dendrite client for {switch}" - )) - })?; - - dpd_client - .ensure_nat_entry_deleted( - &self.log, - external_ip.ip, - *external_ip.first_port, - ) - .await - .map_err(|e| { - Error::internal_error(&format!( - "failed to delete nat entry via dpd: {e}" - )) - })?; - } + "switch" => switch.to_string()); + + let dpd_client = self.dpd_clients.get(switch).ok_or_else(|| { + Error::internal_error(&format!( + "unable to find dendrite client for {switch}" + )) + })?; + + // Notify dendrite that there are changes for it to reconcile. + // In the event of a failure to notify dendrite, we'll log an error + // and rely on dendrite's RPW timer to catch it up. + if let Err(e) = dpd_client.ipv4_nat_trigger_update().await { + error!(self.log, "failed to notify dendrite of nat updates"; "error" => ?e); + }; } Ok(()) diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index ef8132451a..18c9dae841 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -349,6 +349,7 @@ impl Nexus { &background_ctx, Arc::clone(&db_datastore), &config.pkg.background_tasks, + &dpd_clients, config.deployment.id, resolver.clone(), ); diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 163f3bd5bb..1c2e49e260 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -9,6 +9,7 @@ use crate::external_api::params; use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; +use gateway_client::types::SpType; use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; @@ -31,6 +32,9 @@ use nexus_types::external_api::params::{ use nexus_types::external_api::shared::FleetRole; use nexus_types::external_api::shared::SiloIdentityMode; use nexus_types::external_api::shared::SiloRole; +use nexus_types::external_api::views; +use nexus_types::external_api::views::Baseboard; +use nexus_types::external_api::views::UninitializedSled; use nexus_types::internal_api::params::DnsRecord; use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::DataPageParams; @@ -51,6 +55,7 @@ use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; use std::net::Ipv4Addr; +use std::num::NonZeroU32; use std::str::FromStr; use uuid::Uuid; @@ -614,20 +619,7 @@ impl super::Nexus { opctx: &OpContext, ) -> Result { let rack = self.rack_lookup(opctx, &self.rack_id).await?; - - let subnet = match rack.rack_subnet { - Some(IpNetwork::V6(subnet)) => subnet, - Some(IpNetwork::V4(_)) => { - return Err(Error::InternalError { - internal_message: "rack subnet not IPv6".into(), - }) - } - None => { - return Err(Error::InternalError { - internal_message: "rack subnet not set".into(), - }) - } - }; + let subnet = rack.subnet()?; let db_ports = self.active_port_settings(opctx).await?; let mut ports = Vec::new(); @@ -724,4 +716,55 @@ impl super::Nexus { Ok(result) } + + /// Return the list of sleds that are inserted into an initialized rack + /// but not yet initialized as part of a rack. + // + // TODO-multirack: We currently limit sleds to a single rack and we also + // retrieve the `rack_uuid` from the Nexus instance used. + pub(crate) async fn uninitialized_sled_list( + &self, + opctx: &OpContext, + ) -> ListResultVec { + // Grab the SPs from the last collection + let limit = NonZeroU32::new(50).unwrap(); + let collection = self + .db_datastore + .inventory_get_latest_collection(opctx, limit) + .await?; + let pagparams = DataPageParams { + marker: None, + direction: dropshot::PaginationOrder::Descending, + // TODO: This limit is only suitable for a single sled cluster + limit: NonZeroU32::new(32).unwrap(), + }; + let sleds = self.db_datastore.sled_list(opctx, &pagparams).await?; + + let mut uninitialized_sleds: Vec = collection + .sps + .into_iter() + .filter_map(|(k, v)| { + if v.sp_type == SpType::Sled { + Some(UninitializedSled { + baseboard: Baseboard { + serial: k.serial_number.clone(), + part: k.part_number.clone(), + revision: v.baseboard_revision.into(), + }, + rack_id: self.rack_id, + cubby: v.sp_slot, + }) + } else { + None + } + }) + .collect(); + + let sled_baseboards: BTreeSet = + sleds.into_iter().map(|s| views::Sled::from(s).baseboard).collect(); + + // Retain all sleds that exist but are not in the sled table + uninitialized_sleds.retain(|s| !sled_baseboards.contains(&s.baseboard)); + Ok(uninitialized_sleds) + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index eba97a88ec..428632bcf5 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -8,8 +8,8 @@ use super::{ console_api, device_auth, params, views::{ self, Certificate, Group, IdentityProvider, Image, IpPool, IpPoolRange, - PhysicalDisk, Project, Rack, Role, Silo, Sled, Snapshot, SshKey, User, - UserBuiltin, Vpc, VpcRouter, VpcSubnet, + PhysicalDisk, Project, Rack, Role, Silo, Sled, Snapshot, SshKey, + UninitializedSled, User, UserBuiltin, Vpc, VpcRouter, VpcSubnet, }, }; use crate::external_api::shared; @@ -222,6 +222,7 @@ pub(crate) fn external_api() -> NexusApiDescription { api.register(physical_disk_list)?; api.register(switch_list)?; api.register(switch_view)?; + api.register(uninitialized_sled_list)?; api.register(user_builtin_list)?; api.register(user_builtin_view)?; @@ -4382,6 +4383,25 @@ async fn rack_view( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } +/// List uninitialized sleds in a given rack +#[endpoint { + method = GET, + path = "/v1/system/hardware/uninitialized-sleds", + tags = ["system/hardware"] +}] +async fn uninitialized_sled_list( + rqctx: RequestContext>, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let sleds = nexus.uninitialized_sled_list(&opctx).await?; + Ok(HttpResponseOk(sleds)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + // Sleds /// List sleds diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs index ebb21feb40..9a20911893 100644 --- a/nexus/src/internal_api/http_entrypoints.rs +++ b/nexus/src/internal_api/http_entrypoints.rs @@ -24,6 +24,7 @@ use dropshot::RequestContext; use dropshot::ResultsPage; use dropshot::TypedBody; use hyper::Body; +use nexus_db_model::Ipv4NatEntryView; use nexus_types::internal_api::params::SwitchPutRequest; use nexus_types::internal_api::params::SwitchPutResponse; use nexus_types::internal_api::views::to_list; @@ -68,6 +69,8 @@ pub(crate) fn internal_api() -> NexusApiDescription { api.register(saga_list)?; api.register(saga_view)?; + api.register(ipv4_nat_changeset)?; + api.register(bgtask_list)?; api.register(bgtask_view)?; @@ -540,3 +543,51 @@ async fn bgtask_view( }; apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await } + +// NAT RPW internal APIs + +/// Path parameters for NAT ChangeSet +#[derive(Deserialize, JsonSchema)] +struct RpwNatPathParam { + /// which change number to start generating + /// the change set from + from_gen: i64, +} + +/// Query parameters for NAT ChangeSet +#[derive(Deserialize, JsonSchema)] +struct RpwNatQueryParam { + limit: u32, +} + +/// Fetch NAT ChangeSet +/// +/// Caller provides their generation as `from_gen`, along with a query +/// parameter for the page size (`limit`). Endpoint will return changes +/// that have occured since the caller's generation number up to the latest +/// change or until the `limit` is reached. If there are no changes, an +/// empty vec is returned. +#[endpoint { + method = GET, + path = "/nat/ipv4/changeset/{from_gen}" +}] +async fn ipv4_nat_changeset( + rqctx: RequestContext>, + path_params: Path, + query_params: Query, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = crate::context::op_context_for_internal_api(&rqctx).await; + let nexus = &apictx.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let mut changeset = nexus + .datastore() + .ipv4_nat_changeset(&opctx, path.from_gen, query.limit) + .await?; + changeset.sort_by_key(|e| e.gen); + Ok(HttpResponseOk(changeset)) + }; + apictx.internal_latencies.instrument_dropshot_handler(&rqctx, handler).await +} diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 54f7e03eef..fbed9aed8e 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -90,6 +90,7 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 +nat_cleanup.period_secs = 30 # How frequently to collect hardware/software inventory from the whole system # (even if we don't have reason to believe anything has changed). inventory.period_secs = 600 diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 8fba22fb2f..64790c49c2 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -43,6 +43,8 @@ use std::str::FromStr; lazy_static! { pub static ref HARDWARE_RACK_URL: String = format!("/v1/system/hardware/racks/{}", RACK_UUID); + pub static ref HARDWARE_UNINITIALIZED_SLEDS: String = + format!("/v1/system/hardware/uninitialized-sleds"); pub static ref HARDWARE_SLED_URL: String = format!("/v1/system/hardware/sleds/{}", SLED_AGENT_UUID); pub static ref HARDWARE_SWITCH_URL: String = @@ -1564,6 +1566,13 @@ lazy_static! { allowed_methods: vec![AllowedMethod::Get], }, + VerifyEndpoint { + url: &HARDWARE_UNINITIALIZED_SLEDS, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![AllowedMethod::Get], + }, + VerifyEndpoint { url: "/v1/system/hardware/sleds", visibility: Visibility::Public, diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 8c5fe953e3..7f0c30c471 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -122,6 +122,7 @@ sled_physical_disk_list GET /v1/system/hardware/sleds/{sle sled_view GET /v1/system/hardware/sleds/{sled_id} switch_list GET /v1/system/hardware/switches switch_view GET /v1/system/hardware/switches/{switch_id} +uninitialized_sled_list GET /v1/system/hardware/uninitialized-sleds API operations found with tag "system/metrics" OPERATION ID METHOD URL PATH diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index ef3835c618..b34fc7a542 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -274,10 +274,38 @@ pub struct Rack { pub identity: AssetIdentityMetadata, } +/// View of a sled that has not been added to an initialized rack yet +#[derive( + Clone, + Debug, + Serialize, + Deserialize, + JsonSchema, + PartialOrd, + Ord, + PartialEq, + Eq, +)] +pub struct UninitializedSled { + pub baseboard: Baseboard, + pub rack_id: Uuid, + pub cubby: u16, +} + // FRUs /// Properties that uniquely identify an Oxide hardware component -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +#[derive( + Clone, + Debug, + Serialize, + Deserialize, + JsonSchema, + PartialOrd, + Ord, + PartialEq, + Eq, +)] pub struct Baseboard { pub serial: String, pub part: String, diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index f83cf68a8a..fcb285d9eb 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -323,6 +323,57 @@ } } }, + "/nat/ipv4/changeset/{from_gen}": { + "get": { + "summary": "Fetch NAT ChangeSet", + "description": "Caller provides their generation as `from_gen`, along with a query parameter for the page size (`limit`). Endpoint will return changes that have occured since the caller's generation number up to the latest change or until the `limit` is reached. If there are no changes, an empty vec is returned.", + "operationId": "ipv4_nat_changeset", + "parameters": [ + { + "in": "path", + "name": "from_gen", + "description": "which change number to start generating the change set from", + "required": true, + "schema": { + "type": "integer", + "format": "int64" + } + }, + { + "in": "query", + "name": "limit", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Ipv4NatEntryView", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4NatEntryView" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/physical-disk": { "put": { "summary": "Report that a physical disk for the specified sled has come online.", @@ -3763,6 +3814,53 @@ } ] }, + "Ipv4NatEntryView": { + "description": "NAT Record", + "type": "object", + "properties": { + "deleted": { + "type": "boolean" + }, + "external_address": { + "type": "string", + "format": "ipv4" + }, + "first_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "gen": { + "type": "integer", + "format": "int64" + }, + "last_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "mac": { + "$ref": "#/components/schemas/MacAddr" + }, + "sled_address": { + "type": "string", + "format": "ipv6" + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "deleted", + "external_address", + "first_port", + "gen", + "last_port", + "mac", + "sled_address", + "vni" + ] + }, "Ipv4Network": { "type": "string", "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\/(3[0-2]|[0-2]?[0-9])$" @@ -5335,6 +5433,12 @@ "time_updated" ] }, + "Vni": { + "description": "A Geneve Virtual Network Identifier", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "ZpoolPutRequest": { "description": "Sent by a sled agent on startup to Nexus to request further instruction", "type": "object", diff --git a/openapi/nexus.json b/openapi/nexus.json index 74162a9b2b..0d19e81d9a 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -4064,6 +4064,37 @@ } } }, + "/v1/system/hardware/uninitialized-sleds": { + "get": { + "tags": [ + "system/hardware" + ], + "summary": "List uninitialized sleds in a given rack", + "operationId": "uninitialized_sled_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_UninitializedSled", + "type": "array", + "items": { + "$ref": "#/components/schemas/UninitializedSled" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/system/identity-providers": { "get": { "tags": [ @@ -13939,6 +13970,29 @@ "vlan_id" ] }, + "UninitializedSled": { + "description": "View of a sled that has not been added to an initialized rack yet", + "type": "object", + "properties": { + "baseboard": { + "$ref": "#/components/schemas/Baseboard" + }, + "cubby": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "rack_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "baseboard", + "cubby", + "rack_id" + ] + }, "User": { "description": "View of a User", "type": "object", diff --git a/package-manifest.toml b/package-manifest.toml index f320215a13..ca96341f2a 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -476,8 +476,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "147b03901aa8305b5271e0133a09f628b8140949" -source.sha256 = "14fe7f904f963b50188d6e060106b63df6d061ca64238f7b21623c432b5944e3" +source.commit = "8ff834e7d0a6adb263240edd40537f2c0768f1a4" +source.sha256 = "c00e79f55e0bdf048069b2d18a4d009ddfef46e7e5d846887cf96e843a8884bd" output.type = "zone" output.intermediate_only = true @@ -501,8 +501,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "147b03901aa8305b5271e0133a09f628b8140949" -source.sha256 = "f3aa685e4096f8f6e2ea6c169f391dbb88707abcbf1d2bde29163d81736e8ec6" +source.commit = "8ff834e7d0a6adb263240edd40537f2c0768f1a4" +source.sha256 = "428cce1e9aa399b1b49c04e7fd0bc1cb0e3f3fae6fda96055892a42e010c9d6f" output.type = "zone" output.intermediate_only = true @@ -519,8 +519,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "147b03901aa8305b5271e0133a09f628b8140949" -source.sha256 = "dece729ce4127216fba48e9cfed90ec2e5a57ee4ca6c4afc5fa770de6ea636bf" +source.commit = "8ff834e7d0a6adb263240edd40537f2c0768f1a4" +source.sha256 = "5dd3534bec5eb4f857d0bf3994b26650288f650d409eec6aaa29860a2f481c37" output.type = "zone" output.intermediate_only = true diff --git a/package/src/dot.rs b/package/src/dot.rs index f6ac32aa3a..133d5c0f00 100644 --- a/package/src/dot.rs +++ b/package/src/dot.rs @@ -184,7 +184,7 @@ pub fn do_dot( } } - PackageSource::Local { blobs, rust, paths } => { + PackageSource::Local { blobs, rust, paths, .. } => { // Regardless of the type of local package (e.g., files-only or // Rust package or whatever), create nodes showing any S3 blobs // on which it depends. diff --git a/schema/crdb/10.0.0/README.md b/schema/crdb/11.0.0/README.md similarity index 100% rename from schema/crdb/10.0.0/README.md rename to schema/crdb/11.0.0/README.md diff --git a/schema/crdb/10.0.0/up01.sql b/schema/crdb/11.0.0/up01.sql similarity index 100% rename from schema/crdb/10.0.0/up01.sql rename to schema/crdb/11.0.0/up01.sql diff --git a/schema/crdb/10.0.0/up02.sql b/schema/crdb/11.0.0/up02.sql similarity index 100% rename from schema/crdb/10.0.0/up02.sql rename to schema/crdb/11.0.0/up02.sql diff --git a/schema/crdb/10.0.0/up03.sql b/schema/crdb/11.0.0/up03.sql similarity index 100% rename from schema/crdb/10.0.0/up03.sql rename to schema/crdb/11.0.0/up03.sql diff --git a/schema/crdb/10.0.0/up04.sql b/schema/crdb/11.0.0/up04.sql similarity index 100% rename from schema/crdb/10.0.0/up04.sql rename to schema/crdb/11.0.0/up04.sql diff --git a/schema/crdb/10.0.0/up05.sql b/schema/crdb/11.0.0/up05.sql similarity index 100% rename from schema/crdb/10.0.0/up05.sql rename to schema/crdb/11.0.0/up05.sql diff --git a/schema/crdb/10.0.0/up06.sql b/schema/crdb/11.0.0/up06.sql similarity index 100% rename from schema/crdb/10.0.0/up06.sql rename to schema/crdb/11.0.0/up06.sql diff --git a/schema/crdb/10.0.0/up07.sql b/schema/crdb/11.0.0/up07.sql similarity index 100% rename from schema/crdb/10.0.0/up07.sql rename to schema/crdb/11.0.0/up07.sql diff --git a/schema/crdb/10.0.0/up08.sql b/schema/crdb/11.0.0/up08.sql similarity index 100% rename from schema/crdb/10.0.0/up08.sql rename to schema/crdb/11.0.0/up08.sql diff --git a/schema/crdb/10.0.0/up09.sql b/schema/crdb/11.0.0/up09.sql similarity index 100% rename from schema/crdb/10.0.0/up09.sql rename to schema/crdb/11.0.0/up09.sql diff --git a/schema/crdb/11.0.0/up1.sql b/schema/crdb/11.0.0/up1.sql new file mode 100644 index 0000000000..a4d31edd71 --- /dev/null +++ b/schema/crdb/11.0.0/up1.sql @@ -0,0 +1 @@ +CREATE SEQUENCE IF NOT EXISTS omicron.public.ipv4_nat_version START 1 INCREMENT 1; diff --git a/schema/crdb/10.0.0/up10.sql b/schema/crdb/11.0.0/up10.sql similarity index 100% rename from schema/crdb/10.0.0/up10.sql rename to schema/crdb/11.0.0/up10.sql diff --git a/schema/crdb/10.0.0/up11.sql b/schema/crdb/11.0.0/up11.sql similarity index 100% rename from schema/crdb/10.0.0/up11.sql rename to schema/crdb/11.0.0/up11.sql diff --git a/schema/crdb/10.0.0/up12.sql b/schema/crdb/11.0.0/up12.sql similarity index 100% rename from schema/crdb/10.0.0/up12.sql rename to schema/crdb/11.0.0/up12.sql diff --git a/schema/crdb/11.0.0/up2.sql b/schema/crdb/11.0.0/up2.sql new file mode 100644 index 0000000000..b92d4c73d3 --- /dev/null +++ b/schema/crdb/11.0.0/up2.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.ipv4_nat_entry ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_address INET NOT NULL, + first_port INT4 NOT NULL, + last_port INT4 NOT NULL, + sled_address INET NOT NULL, + vni INT4 NOT NULL, + mac INT8 NOT NULL, + version_added INT8 NOT NULL DEFAULT nextval('omicron.public.ipv4_nat_version'), + version_removed INT8, + time_created TIMESTAMPTZ NOT NULL DEFAULT now(), + time_deleted TIMESTAMPTZ +); diff --git a/schema/crdb/11.0.0/up3.sql b/schema/crdb/11.0.0/up3.sql new file mode 100644 index 0000000000..1247aad693 --- /dev/null +++ b/schema/crdb/11.0.0/up3.sql @@ -0,0 +1,13 @@ +CREATE UNIQUE INDEX IF NOT EXISTS ipv4_nat_version_added ON omicron.public.ipv4_nat_entry ( + version_added +) +STORING ( + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + time_created, + time_deleted +); diff --git a/schema/crdb/11.0.0/up4.sql b/schema/crdb/11.0.0/up4.sql new file mode 100644 index 0000000000..b9cfe305d2 --- /dev/null +++ b/schema/crdb/11.0.0/up4.sql @@ -0,0 +1,5 @@ +CREATE UNIQUE INDEX IF NOT EXISTS overlapping_ipv4_nat_entry ON omicron.public.ipv4_nat_entry ( + external_address, + first_port, + last_port +) WHERE time_deleted IS NULL; diff --git a/schema/crdb/11.0.0/up5.sql b/schema/crdb/11.0.0/up5.sql new file mode 100644 index 0000000000..dce2211eae --- /dev/null +++ b/schema/crdb/11.0.0/up5.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS ipv4_nat_lookup ON omicron.public.ipv4_nat_entry (external_address, first_port, last_port, sled_address, vni, mac); diff --git a/schema/crdb/11.0.0/up6.sql b/schema/crdb/11.0.0/up6.sql new file mode 100644 index 0000000000..e4958eb352 --- /dev/null +++ b/schema/crdb/11.0.0/up6.sql @@ -0,0 +1,13 @@ +CREATE UNIQUE INDEX IF NOT EXISTS ipv4_nat_version_removed ON omicron.public.ipv4_nat_entry ( + version_removed +) +STORING ( + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + time_created, + time_deleted +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 875877ee96..a74cabfe6e 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -2738,12 +2738,24 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_caboose ( COMMIT; BEGIN; -/*******************************************************************/ +CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( + -- There should only be one row of this table for the whole DB. + -- It's a little goofy, but filter on "singleton = true" before querying + -- or applying updates, and you'll access the singleton row. + -- + -- We also add a constraint on this table to ensure it's not possible to + -- access the version of this table with "singleton = false". + singleton BOOL NOT NULL PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + -- Semver representation of the DB version + version STRING(64) NOT NULL, -/* - * Metadata for the schema itself. This version number isn't great, as there's - * nothing to ensure it gets bumped when it should be, but it's a start. - */ + -- (Optional) Semver representation of the DB version to which we're upgrading + target_version STRING(64), + + CHECK (singleton = true) +); -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( @@ -2812,6 +2824,62 @@ CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; +CREATE SEQUENCE IF NOT EXISTS omicron.public.ipv4_nat_version START 1 INCREMENT 1; + +CREATE TABLE IF NOT EXISTS omicron.public.ipv4_nat_entry ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + external_address INET NOT NULL, + first_port INT4 NOT NULL, + last_port INT4 NOT NULL, + sled_address INET NOT NULL, + vni INT4 NOT NULL, + mac INT8 NOT NULL, + version_added INT8 NOT NULL DEFAULT nextval('omicron.public.ipv4_nat_version'), + version_removed INT8, + time_created TIMESTAMPTZ NOT NULL DEFAULT now(), + time_deleted TIMESTAMPTZ +); + +CREATE UNIQUE INDEX IF NOT EXISTS ipv4_nat_version_added ON omicron.public.ipv4_nat_entry ( + version_added +) +STORING ( + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + time_created, + time_deleted +); + +CREATE UNIQUE INDEX IF NOT EXISTS overlapping_ipv4_nat_entry ON omicron.public.ipv4_nat_entry ( + external_address, + first_port, + last_port +) WHERE time_deleted IS NULL; + +CREATE INDEX IF NOT EXISTS ipv4_nat_lookup ON omicron.public.ipv4_nat_entry (external_address, first_port, last_port, sled_address, vni, mac); + +CREATE UNIQUE INDEX IF NOT EXISTS ipv4_nat_version_removed ON omicron.public.ipv4_nat_entry ( + version_removed +) +STORING ( + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + time_created, + time_deleted +); + +/* + * Metadata for the schema itself. This version number isn't great, as there's + * nothing to ensure it gets bumped when it should be, but it's a start. + */ CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( -- There should only be one row of this table for the whole DB. -- It's a little goofy, but filter on "singleton = true" before querying @@ -2838,7 +2906,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '10.0.0', NULL) + ( TRUE, NOW(), NOW(), '11.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index cae1f650c9..94c8f5572e 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -38,6 +38,7 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 +nat_cleanup.period_secs = 30 # How frequently to collect hardware/software inventory from the whole system # (even if we don't have reason to believe anything has changed). inventory.period_secs = 600 diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index be8683be54..fcaa6176a8 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -38,6 +38,7 @@ dns_external.max_concurrent_server_updates = 5 # certificates it will take _other_ Nexus instances to notice and stop serving # them (on a sunny day). external_endpoints.period_secs = 60 +nat_cleanup.period_secs = 30 # How frequently to collect hardware/software inventory from the whole system # (even if we don't have reason to believe anything has changed). inventory.period_secs = 600 diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index aadf68da1b..ba4b5a5722 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="147b03901aa8305b5271e0133a09f628b8140949" -SHA2="82437c74afd4894aa5b9ea800d5777793e8777fe87471321dd22ad1a1c9c9ef3" +COMMIT="8ff834e7d0a6adb263240edd40537f2c0768f1a4" +SHA2="07d115bfa8498a8015ca2a8447efeeac32e24aeb25baf3d5e2313216e11293c0" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index 81a957323c..619a6bf287 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="14fe7f904f963b50188d6e060106b63df6d061ca64238f7b21623c432b5944e3" -CIDL_SHA256_LINUX_DPD="fff6c7484bbb06aa644e3fe41b200e4f7f8d7f65d067cbecd851c834c15fe2ec" -CIDL_SHA256_LINUX_SWADM="0449383a57468aec3b5a4ad26962cfc9e9a121bd13e777329e8a70767e6d9aae" +CIDL_SHA256_ILLUMOS="c00e79f55e0bdf048069b2d18a4d009ddfef46e7e5d846887cf96e843a8884bd" +CIDL_SHA256_LINUX_DPD="b5d829b4628759ac374106f3c56c29074b29577fd0ff72f61c3b8289fea430fe" +CIDL_SHA256_LINUX_SWADM="afc68828f54dc57b32dc1556fc588baeab12341c30e96cc0fadb49f401b4b48f" diff --git a/tools/reflector/helpers.sh b/tools/reflector/helpers.sh index 92d132faae..3d4f693da2 100644 --- a/tools/reflector/helpers.sh +++ b/tools/reflector/helpers.sh @@ -19,7 +19,7 @@ function merge { local TARGET_BRANCH="$1" local INTEGRATION_BRANCH="$2" local BOT_ID="$3" - local -n CHECKOUT_PATHS=$4 + local CHECKOUT_PATHS=$4 set_reflector_bot "$BOT_ID" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 86c61f1ca8..47ea83f8f2 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -33,7 +33,7 @@ crossbeam-epoch = { version = "0.9.15" } crossbeam-utils = { version = "0.8.16" } crossterm = { version = "0.27.0", features = ["event-stream", "serde"] } crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } -diesel = { version = "2.1.3", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } +diesel = { version = "2.1.4", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0" } flate2 = { version = "1.0.28" } @@ -128,7 +128,7 @@ crossbeam-epoch = { version = "0.9.15" } crossbeam-utils = { version = "0.8.16" } crossterm = { version = "0.27.0", features = ["event-stream", "serde"] } crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } -diesel = { version = "2.1.3", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } +diesel = { version = "2.1.4", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0" } flate2 = { version = "1.0.28" }