diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index a9f1bf8a9c..502909ec2d 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -340,7 +340,7 @@ E2E_TLS_CERT="/opt/oxide/sled-agent/pkg/initial-tls-cert.pem" # pfexec mkdir -p /usr/oxide pfexec curl -sSfL -o /usr/oxide/oxide \ - http://catacomb.eng.oxide.computer:12346/oxide-v0.1.0 + http://catacomb.eng.oxide.computer:12346/oxide-v0.1.1 pfexec chmod +x /usr/oxide/oxide curl -sSfL -o debian-11-genericcloud-amd64.raw \ @@ -418,11 +418,9 @@ done /usr/oxide/oxide --resolve "$OXIDE_RESOLVE" --cacert "$E2E_TLS_CERT" \ project create --name images --description "some images" -# NOTE: Use a relatively large timeout on this call, to avoid #6771 /usr/oxide/oxide \ --resolve "$OXIDE_RESOLVE" \ --cacert "$E2E_TLS_CERT" \ - --timeout 60 \ disk import \ --path debian-11-genericcloud-amd64.raw \ --disk debian11-boot \ @@ -432,7 +430,8 @@ done --image debian11 \ --image-description "debian 11 original base image" \ --image-os debian \ - --image-version "11" + --image-version "11" \ + --parallelism 1 /usr/oxide/oxide --resolve "$OXIDE_RESOLVE" --cacert "$E2E_TLS_CERT" \ image promote --project images --image debian11 diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index 7e9f3662e4..6c90f6a170 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -16,14 +16,14 @@ jobs: env: RUSTFLAGS: -D warnings steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@939f4af9602e15ef93b3329722569cb907a004ff # v2 + uses: taiki-e/install-action@4b40a9728e3c110fabe8850443d8bbe69daddb22 # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date diff --git a/Cargo.lock b/Cargo.lock index d3370fe34a..77f6a20491 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" dependencies = [ "backtrace", ] @@ -178,6 +178,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -656,7 +665,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927#11371b0f3743f8df5b047dc0edc2699f4bdf3927" +source = "git+https://github.com/oxidecomputer/propolis?rev=86101eaf80b55e7f405b5cafe9b0de0e9f331656#86101eaf80b55e7f405b5cafe9b0de0e9f331656" dependencies = [ "bhyve_api_sys", "libc", @@ -666,7 +675,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927#11371b0f3743f8df5b047dc0edc2699f4bdf3927" +source = "git+https://github.com/oxidecomputer/propolis?rev=86101eaf80b55e7f405b5cafe9b0de0e9f331656#86101eaf80b55e7f405b5cafe9b0de0e9f331656" dependencies = [ "libc", "strum", @@ -1735,7 +1744,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=2b88ab88461fb06aaf2aab11c5e381a3cad25eac#2b88ab88461fb06aaf2aab11c5e381a3cad25eac" +source = "git+https://github.com/oxidecomputer/crucible?rev=b7b9d5660b28ca5e865242b2bdecd032c0852d40#b7b9d5660b28ca5e865242b2bdecd032c0852d40" dependencies = [ "anyhow", "chrono", @@ -1751,7 +1760,7 @@ dependencies = [ [[package]] name = "crucible-client-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=2b88ab88461fb06aaf2aab11c5e381a3cad25eac#2b88ab88461fb06aaf2aab11c5e381a3cad25eac" +source = "git+https://github.com/oxidecomputer/crucible?rev=b7b9d5660b28ca5e865242b2bdecd032c0852d40#b7b9d5660b28ca5e865242b2bdecd032c0852d40" dependencies = [ "base64 0.22.1", "crucible-workspace-hack", @@ -1764,7 +1773,7 @@ dependencies = [ [[package]] name = "crucible-common" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=2b88ab88461fb06aaf2aab11c5e381a3cad25eac#2b88ab88461fb06aaf2aab11c5e381a3cad25eac" +source = "git+https://github.com/oxidecomputer/crucible?rev=b7b9d5660b28ca5e865242b2bdecd032c0852d40#b7b9d5660b28ca5e865242b2bdecd032c0852d40" dependencies = [ "anyhow", "atty", @@ -1794,7 +1803,7 @@ dependencies = [ [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=2b88ab88461fb06aaf2aab11c5e381a3cad25eac#2b88ab88461fb06aaf2aab11c5e381a3cad25eac" +source = "git+https://github.com/oxidecomputer/crucible?rev=b7b9d5660b28ca5e865242b2bdecd032c0852d40#b7b9d5660b28ca5e865242b2bdecd032c0852d40" dependencies = [ "anyhow", "chrono", @@ -1811,7 +1820,7 @@ dependencies = [ [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=2b88ab88461fb06aaf2aab11c5e381a3cad25eac#2b88ab88461fb06aaf2aab11c5e381a3cad25eac" +source = "git+https://github.com/oxidecomputer/crucible?rev=b7b9d5660b28ca5e865242b2bdecd032c0852d40#b7b9d5660b28ca5e865242b2bdecd032c0852d40" dependencies = [ "crucible-workspace-hack", "libc", @@ -2111,6 +2120,17 @@ dependencies = [ "syn 2.0.79", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "derive_builder" version = "0.20.2" @@ -2303,9 +2323,20 @@ dependencies = [ [[package]] name = "display-error-chain" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d305e5a3904ee14166439a70feef04853c1234226dbb27ede127b88dc5a4a9d" +checksum = "0bc2146e86bc19f52f4c064a64782f05f139ca464ed72937301631e73f8d6cf5" + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] [[package]] name = "dlpi" @@ -3248,7 +3279,7 @@ dependencies = [ "usdt", "uuid", "version_check", - "zip", + "zip 0.6.6", ] [[package]] @@ -3957,7 +3988,7 @@ dependencies = [ "toml 0.7.8", "x509-cert", "zerocopy 0.6.6", - "zip", + "zip 0.6.6", ] [[package]] @@ -4805,7 +4836,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -4978,6 +5009,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.22" @@ -5749,6 +5786,7 @@ dependencies = [ "debug-ignore", "expectorate", "gateway-client", + "illumos-utils", "indexmap 2.6.0", "internal-dns-resolver", "ipnet", @@ -6637,7 +6675,7 @@ dependencies = [ "pq-sys", "pretty_assertions", "progenitor-client", - "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927)", + "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=86101eaf80b55e7f405b5cafe9b0de0e9f331656)", "qorb", "rand", "rcgen", @@ -6870,6 +6908,7 @@ dependencies = [ "glob", "guppy", "hex", + "hex-literal", "http 1.1.0", "hyper 1.4.1", "hyper-staticfile", @@ -6899,17 +6938,20 @@ dependencies = [ "oximeter-producer", "oxnet", "pretty_assertions", - "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927)", + "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=86101eaf80b55e7f405b5cafe9b0de0e9f331656)", "propolis-mock-server", "propolis_api_types", "rand", "rcgen", + "repo-depot-api", + "repo-depot-client", "reqwest 0.12.8", "schemars", "semver 1.0.23", "serde", "serde_human_bytes", "serde_json", + "sha2", "sha3", "sled-agent-api", "sled-agent-client", @@ -6970,6 +7012,7 @@ dependencies = [ "reqwest 0.12.8", "ring 0.17.8", "rustls 0.22.4", + "serde", "slog", "subprocess", "tar", @@ -7124,6 +7167,7 @@ dependencies = [ "x509-cert", "zerocopy 0.7.35", "zeroize", + "zip 0.6.6", ] [[package]] @@ -7213,6 +7257,7 @@ dependencies = [ "openapiv3", "owo-colors", "oximeter-api", + "repo-depot-api", "serde_json", "similar", "sled-agent-api", @@ -7876,9 +7921,9 @@ dependencies = [ [[package]] name = "parse-size" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "944553dd59c802559559161f9816429058b869003836120e262e8caec061b7ae" +checksum = "487f2ccd1e17ce8c1bfab3a65c89525af41cfad4c8659021a1e9a2aacd73b89b" [[package]] name = "parse-zoneinfo" @@ -8583,7 +8628,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927#11371b0f3743f8df5b047dc0edc2699f4bdf3927" +source = "git+https://github.com/oxidecomputer/propolis?rev=86101eaf80b55e7f405b5cafe9b0de0e9f331656#86101eaf80b55e7f405b5cafe9b0de0e9f331656" dependencies = [ "async-trait", "base64 0.21.7", @@ -8625,7 +8670,7 @@ dependencies = [ [[package]] name = "propolis-mock-server" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927#11371b0f3743f8df5b047dc0edc2699f4bdf3927" +source = "git+https://github.com/oxidecomputer/propolis?rev=86101eaf80b55e7f405b5cafe9b0de0e9f331656#86101eaf80b55e7f405b5cafe9b0de0e9f331656" dependencies = [ "anyhow", "atty", @@ -8667,7 +8712,7 @@ dependencies = [ [[package]] name = "propolis_api_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927#11371b0f3743f8df5b047dc0edc2699f4bdf3927" +source = "git+https://github.com/oxidecomputer/propolis?rev=86101eaf80b55e7f405b5cafe9b0de0e9f331656#86101eaf80b55e7f405b5cafe9b0de0e9f331656" dependencies = [ "crucible-client-types", "propolis_types", @@ -8680,7 +8725,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927#11371b0f3743f8df5b047dc0edc2699f4bdf3927" +source = "git+https://github.com/oxidecomputer/propolis?rev=86101eaf80b55e7f405b5cafe9b0de0e9f331656#86101eaf80b55e7f405b5cafe9b0de0e9f331656" dependencies = [ "schemars", "serde", @@ -8724,9 +8769,9 @@ dependencies = [ [[package]] name = "qorb" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25f71eb7c5ba56a99f0721fd771b2503aa6de4ec73f0891f9b7ac115ca34723" +checksum = "9cd19ad8fae9abd8da01d8f435b633b567d53835cf3bce89d6f616617d10583c" dependencies = [ "anyhow", "async-trait", @@ -8744,6 +8789,7 @@ dependencies = [ "tokio-stream", "tokio-tungstenite 0.24.0", "tracing", + "usdt", ] [[package]] @@ -9131,6 +9177,29 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "repo-depot-api" +version = "0.1.0" +dependencies = [ + "dropshot", + "omicron-common", + "omicron-workspace-hack", + "schemars", + "serde", +] + +[[package]] +name = "repo-depot-client" +version = "0.1.0" +dependencies = [ + "omicron-workspace-hack", + "progenitor", + "reqwest 0.12.8", + "schemars", + "serde", + "slog", +] + [[package]] name = "reqwest" version = "0.11.27" @@ -10223,6 +10292,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "similar" version = "2.6.0" @@ -10329,7 +10404,7 @@ dependencies = [ "omicron-uuid-kinds", "omicron-workspace-hack", "oxnet", - "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927)", + "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=86101eaf80b55e7f405b5cafe9b0de0e9f331656)", "rcgen", "schemars", "serde", @@ -11841,7 +11916,7 @@ dependencies = [ "toml 0.8.19", "tough", "url", - "zip", + "zip 2.1.3", ] [[package]] @@ -13178,6 +13253,24 @@ dependencies = [ "flate2", ] +[[package]] +name = "zip" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" +dependencies = [ + "arbitrary", + "bzip2", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap 2.6.0", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zone" version = "0.1.8" @@ -13247,3 +13340,17 @@ dependencies = [ "quote", "syn 1.0.109", ] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index e751c36779..ab82cbef54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "clients/nexus-client", "clients/oxide-client", "clients/oximeter-client", + "clients/repo-depot-client", "clients/sled-agent-client", "clients/wicketd-client", "cockroach-admin", @@ -100,6 +101,7 @@ members = [ "sled-agent", "sled-agent/api", "sled-agent/bootstrap-agent-api", + "sled-agent/repo-depot-api", "sled-agent/types", "sled-hardware", "sled-hardware/types", @@ -140,6 +142,7 @@ default-members = [ "clients/nexus-client", "clients/oxide-client", "clients/oximeter-client", + "clients/repo-depot-client", "clients/sled-agent-client", "clients/wicketd-client", "cockroach-admin", @@ -225,6 +228,7 @@ default-members = [ "sled-agent", "sled-agent/api", "sled-agent/bootstrap-agent-api", + "sled-agent/repo-depot-api", "sled-agent/types", "sled-hardware", "sled-hardware/types", @@ -329,14 +333,14 @@ cookie = "0.18" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.28.1", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "2b88ab88461fb06aaf2aab11c5e381a3cad25eac" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "2b88ab88461fb06aaf2aab11c5e381a3cad25eac" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "2b88ab88461fb06aaf2aab11c5e381a3cad25eac" } -crucible-common = { git = "https://github.com/oxidecomputer/crucible", rev = "2b88ab88461fb06aaf2aab11c5e381a3cad25eac" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "b7b9d5660b28ca5e865242b2bdecd032c0852d40" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "b7b9d5660b28ca5e865242b2bdecd032c0852d40" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "b7b9d5660b28ca5e865242b2bdecd032c0852d40" } +crucible-common = { git = "https://github.com/oxidecomputer/crucible", rev = "b7b9d5660b28ca5e865242b2bdecd032c0852d40" } csv = "1.3.0" curve25519-dalek = "4" datatest-stable = "0.2.9" -display-error-chain = "0.2.1" +display-error-chain = "0.2.2" omicron-ddm-admin-client = { path = "clients/ddm-admin-client" } db-macros = { path = "nexus/db-macros" } debug-ignore = "1.0.5" @@ -515,12 +519,12 @@ prettyplease = { version = "0.2.22", features = ["verbatim"] } proc-macro2 = "1.0" progenitor = "0.8.0" progenitor-client = "0.8.0" -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "11371b0f3743f8df5b047dc0edc2699f4bdf3927" } -propolis_api_types = { git = "https://github.com/oxidecomputer/propolis", rev = "11371b0f3743f8df5b047dc0edc2699f4bdf3927" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "11371b0f3743f8df5b047dc0edc2699f4bdf3927" } -propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "11371b0f3743f8df5b047dc0edc2699f4bdf3927" } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "86101eaf80b55e7f405b5cafe9b0de0e9f331656" } +propolis_api_types = { git = "https://github.com/oxidecomputer/propolis", rev = "86101eaf80b55e7f405b5cafe9b0de0e9f331656" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "86101eaf80b55e7f405b5cafe9b0de0e9f331656" } +propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "86101eaf80b55e7f405b5cafe9b0de0e9f331656" } proptest = "1.5.0" -qorb = "0.1.2" +qorb = "0.2.0" quote = "1.0" rand = "0.8.5" rand_core = "0.6.4" @@ -533,6 +537,8 @@ reedline = "0.35.0" ref-cast = "1.0" regex = "1.11.0" regress = "0.9.1" +repo-depot-api = { path = "sled-agent/repo-depot-api" } +repo-depot-client = { path = "clients/repo-depot-client" } reqwest = { version = "0.12", default-features = false } ring = "0.17.8" rpassword = "7.3.1" @@ -632,7 +638,8 @@ wicket-common = { path = "wicket-common" } wicketd-api = { path = "wicketd-api" } wicketd-client = { path = "clients/wicketd-client" } zeroize = { version = "1.8.1", features = ["zeroize_derive", "std"] } -zip = { version = "0.6.6", default-features = false, features = ["deflate","bzip2"] } +# NOTE: Avoid upgrading zip until https://github.com/zip-rs/zip2/issues/231 is resolved +zip = { version = "=2.1.3", default-features = false, features = ["deflate","bzip2"] } zone = { version = "0.3", default-features = false, features = ["async"] } # newtype-uuid is set to default-features = false because we don't want to diff --git a/clickhouse-admin/api/src/lib.rs b/clickhouse-admin/api/src/lib.rs index 19cd2b3e8e..10ba028016 100644 --- a/clickhouse-admin/api/src/lib.rs +++ b/clickhouse-admin/api/src/lib.rs @@ -3,8 +3,8 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig, - KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, + ClickhouseKeeperClusterMembership, DistributedDdlQueue, KeeperConf, + KeeperConfig, KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, ServerConfigurableSettings, }; use dropshot::{ @@ -105,4 +105,14 @@ pub trait ClickhouseAdminServerApi { rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError>; + + /// Contains information about distributed ddl queries (ON CLUSTER clause) + /// that were executed on a cluster. + #[endpoint { + method = GET, + path = "/distributed-ddl-queue", + }] + async fn distributed_ddl_queue( + rqctx: RequestContext, + ) -> Result>, HttpError>; } diff --git a/clickhouse-admin/src/clickhouse_cli.rs b/clickhouse-admin/src/clickhouse_cli.rs index 32afdc4ef8..440a8f7d9e 100644 --- a/clickhouse-admin/src/clickhouse_cli.rs +++ b/clickhouse-admin/src/clickhouse_cli.rs @@ -5,7 +5,8 @@ use anyhow::Result; use camino::Utf8PathBuf; use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, KeeperConf, KeeperId, Lgif, RaftConfig, + ClickhouseKeeperClusterMembership, DistributedDdlQueue, KeeperConf, + KeeperId, Lgif, RaftConfig, OXIMETER_CLUSTER, }; use dropshot::HttpError; use illumos_utils::{output_to_exec_error, ExecutionError}; @@ -13,6 +14,7 @@ use slog::Logger; use slog_error_chain::{InlineErrorChain, SlogInlineError}; use std::collections::BTreeSet; use std::ffi::OsStr; +use std::fmt::Display; use std::io; use std::net::SocketAddrV6; use tokio::process::Command; @@ -56,6 +58,21 @@ impl From for HttpError { } } +enum ClickhouseClientType { + Server, + Keeper, +} + +impl Display for ClickhouseClientType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + ClickhouseClientType::Server => "client", + ClickhouseClientType::Keeper => "keeper-client", + }; + write!(f, "{s}") + } +} + #[derive(Debug)] pub struct ClickhouseCli { /// Path to where the clickhouse binary is located @@ -76,7 +93,8 @@ impl ClickhouseCli { } pub async fn lgif(&self) -> Result { - self.keeper_client_non_interactive( + self.client_non_interactive( + ClickhouseClientType::Keeper, "lgif", "Retrieve logically grouped information file", Lgif::parse, @@ -86,7 +104,8 @@ impl ClickhouseCli { } pub async fn raft_config(&self) -> Result { - self.keeper_client_non_interactive( + self.client_non_interactive( + ClickhouseClientType::Keeper, "get /keeper/config", "Retrieve raft configuration information", RaftConfig::parse, @@ -96,7 +115,8 @@ impl ClickhouseCli { } pub async fn keeper_conf(&self) -> Result { - self.keeper_client_non_interactive( + self.client_non_interactive( + ClickhouseClientType::Keeper, "conf", "Retrieve keeper node configuration information", KeeperConf::parse, @@ -121,8 +141,26 @@ impl ClickhouseCli { }) } - async fn keeper_client_non_interactive( + pub async fn distributed_ddl_queue( + &self, + ) -> Result, ClickhouseCliError> { + self.client_non_interactive( + ClickhouseClientType::Server, + format!( + "SELECT * FROM system.distributed_ddl_queue WHERE cluster = '{}' FORMAT JSONEachRow", + OXIMETER_CLUSTER + ).as_str(), + "Retrieve information about distributed ddl queries (ON CLUSTER clause) + that were executed on a cluster", + DistributedDdlQueue::parse, + self.log.clone().unwrap(), + ) + .await + } + + async fn client_non_interactive( &self, + client: ClickhouseClientType, query: &str, subcommand_description: &'static str, parse: F, @@ -133,7 +171,7 @@ impl ClickhouseCli { { let mut command = Command::new(&self.binary_path); command - .arg("keeper-client") + .arg(client.to_string()) .arg("--host") .arg(&format!("[{}]", self.listen_address.ip())) .arg("--port") diff --git a/clickhouse-admin/src/http_entrypoints.rs b/clickhouse-admin/src/http_entrypoints.rs index 49138b9cc3..e1974cfc7e 100644 --- a/clickhouse-admin/src/http_entrypoints.rs +++ b/clickhouse-admin/src/http_entrypoints.rs @@ -5,8 +5,8 @@ use crate::context::ServerContext; use clickhouse_admin_api::*; use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig, - KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, + ClickhouseKeeperClusterMembership, DistributedDdlQueue, KeeperConf, + KeeperConfig, KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, ServerConfigurableSettings, }; use dropshot::{ @@ -47,6 +47,14 @@ impl ClickhouseAdminServerApi for ClickhouseAdminServerImpl { Ok(HttpResponseCreated(output)) } + + async fn distributed_ddl_queue( + rqctx: RequestContext, + ) -> Result>, HttpError> { + let ctx = rqctx.context(); + let output = ctx.clickhouse_cli().distributed_ddl_queue().await?; + Ok(HttpResponseOk(output)) + } } enum ClickhouseAdminKeeperImpl {} diff --git a/clickhouse-admin/types/src/config.rs b/clickhouse-admin/types/src/config.rs index 20aa3d71f8..27eb569b91 100644 --- a/clickhouse-admin/types/src/config.rs +++ b/clickhouse-admin/types/src/config.rs @@ -154,6 +154,11 @@ impl ReplicaConfig { 1000 + + + + 1.0 + {macros} {remote_servers} {keepers} diff --git a/clickhouse-admin/types/src/lib.rs b/clickhouse-admin/types/src/lib.rs index 021b9db356..d079cee6ec 100644 --- a/clickhouse-admin/types/src/lib.rs +++ b/clickhouse-admin/types/src/lib.rs @@ -15,7 +15,7 @@ use schemars::{ }; use serde::{Deserialize, Serialize}; use slog::{info, Logger}; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::fs::create_dir; use std::io::{ErrorKind, Write}; use std::net::Ipv6Addr; @@ -965,19 +965,87 @@ pub struct ClickhouseKeeperClusterMembership { pub raft_config: BTreeSet, } +#[derive( + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +#[serde(rename_all = "snake_case")] +/// Contains information about distributed ddl queries (ON CLUSTER clause) that were +/// executed on a cluster. +pub struct DistributedDdlQueue { + /// Query id + pub entry: String, + /// Version of the entry + pub entry_version: u64, + /// Host that initiated the DDL operation + pub initiator_host: String, + /// Port used by the initiator + pub initiator_port: u16, + /// Cluster name + pub cluster: String, + /// Query executed + pub query: String, + /// Settings used in the DDL operation + pub settings: BTreeMap, + /// Query created time + pub query_create_time: String, + /// Hostname + pub host: Ipv6Addr, + /// Host Port + pub port: u16, + /// Status of the query + pub status: String, + /// Exception code + pub exception_code: u64, + /// Exception message + pub exception_text: String, + /// Query finish time + pub query_finish_time: String, + /// Duration of query execution (in milliseconds) + pub query_duration_ms: String, +} + +impl DistributedDdlQueue { + pub fn parse(log: &Logger, data: &[u8]) -> Result> { + let s = String::from_utf8_lossy(data); + info!( + log, + "Retrieved data from `system.distributed_ddl_queue`"; + "output" => ?s + ); + + let mut ddl = vec![]; + + for line in s.lines() { + let item: DistributedDdlQueue = serde_json::from_str(line)?; + ddl.push(item); + } + + Ok(ddl) + } +} + #[cfg(test)] mod tests { use camino::Utf8PathBuf; use camino_tempfile::Builder; use slog::{o, Drain}; use slog_term::{FullFormat, PlainDecorator, TestStdoutWriter}; + use std::collections::BTreeMap; use std::net::{Ipv4Addr, Ipv6Addr}; use std::str::FromStr; use crate::{ - ClickhouseHost, KeeperConf, KeeperId, KeeperServerInfo, - KeeperServerType, KeeperSettings, Lgif, LogLevel, RaftConfig, - RaftServerSettings, ServerId, ServerSettings, + ClickhouseHost, DistributedDdlQueue, KeeperConf, KeeperId, + KeeperServerInfo, KeeperServerType, KeeperSettings, Lgif, LogLevel, + RaftConfig, RaftServerSettings, ServerId, ServerSettings, }; fn log() -> slog::Logger { @@ -1736,4 +1804,85 @@ snapshot_storage_disk=LocalSnapshotDisk "Extracted key `\"session_timeout_fake\"` from output differs from expected key `session_timeout_ms`" ); } + + #[test] + fn test_distributed_ddl_queries_parse_success() { + let log = log(); + let data = + "{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01 16:16:45\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01 16:16:45\",\"query_duration_ms\":\"4\"} +{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01 16:16:45\",\"host\":\"::1\",\"port\":22002,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01 16:16:45\",\"query_duration_ms\":\"4\"} +" + .as_bytes(); + let ddl = DistributedDdlQueue::parse(&log, data).unwrap(); + + let expected_result = vec![ + DistributedDdlQueue{ + entry: "query-0000000000".to_string(), + entry_version: 5, + initiator_host: "ixchel".to_string(), + initiator_port: 22001, + cluster: "oximeter_cluster".to_string(), + query: "CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster".to_string(), + settings: BTreeMap::from([ + ("load_balancing".to_string(), "random".to_string()), +]), + query_create_time: "2024-11-01 16:16:45".to_string(), + host: Ipv6Addr::from_str("::1").unwrap(), + port: 22001, + exception_code: 0, + exception_text: "".to_string(), + status: "Finished".to_string(), + query_finish_time: "2024-11-01 16:16:45".to_string(), + query_duration_ms: "4".to_string(), + }, + DistributedDdlQueue{ + entry: "query-0000000000".to_string(), + entry_version: 5, + initiator_host: "ixchel".to_string(), + initiator_port: 22001, + cluster: "oximeter_cluster".to_string(), + query: "CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster".to_string(), + settings: BTreeMap::from([ + ("load_balancing".to_string(), "random".to_string()), +]), + query_create_time: "2024-11-01 16:16:45".to_string(), + host: Ipv6Addr::from_str("::1").unwrap(), + port: 22002, + exception_code: 0, + exception_text: "".to_string(), + status: "Finished".to_string(), + query_finish_time: "2024-11-01 16:16:45".to_string(), + query_duration_ms: "4".to_string(), + }, + ]; + assert!(ddl == expected_result); + } + + #[test] + fn test_empty_distributed_ddl_queries_parse_success() { + let log = log(); + let data = "".as_bytes(); + let ddl = DistributedDdlQueue::parse(&log, data).unwrap(); + + let expected_result = vec![]; + assert!(ddl == expected_result); + } + + #[test] + fn test_misshapen_distributed_ddl_queries_parse_fail() { + let log = log(); + let data = + "{\"entry\":\"query-0000000000\",\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01 16:16:45\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01 16:16:45\",\"query_duration_ms\":\"4\"} +" +.as_bytes(); + let result = DistributedDdlQueue::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "missing field `entry_version` at line 1 column 454", + ); + } } diff --git a/clickhouse-admin/types/testutils/replica-server-config.xml b/clickhouse-admin/types/testutils/replica-server-config.xml index cd79cf4a68..3aeacd073d 100644 --- a/clickhouse-admin/types/testutils/replica-server-config.xml +++ b/clickhouse-admin/types/testutils/replica-server-config.xml @@ -72,6 +72,11 @@ 1000 + + + 1.0 + + 1 1 diff --git a/clients/clickhouse-admin-server-client/src/lib.rs b/clients/clickhouse-admin-server-client/src/lib.rs index 3092160d65..73fe828824 100644 --- a/clients/clickhouse-admin-server-client/src/lib.rs +++ b/clients/clickhouse-admin-server-client/src/lib.rs @@ -4,6 +4,7 @@ //! Interface for making API requests to a clickhouse-admin-server server //! running in an omicron zone. +use std::clone::Clone; progenitor::generate_api!( spec = "../../openapi/clickhouse-admin-server.json", diff --git a/clients/dpd-client/build.rs b/clients/dpd-client/build.rs index 02a685632c..313c1a452f 100644 --- a/clients/dpd-client/build.rs +++ b/clients/dpd-client/build.rs @@ -15,7 +15,6 @@ use anyhow::Context; use anyhow::Result; use omicron_zone_package::config::Config; use omicron_zone_package::package::PackageSource; -use progenitor::TypePatch; use quote::quote; use std::env; use std::fs; @@ -89,22 +88,7 @@ fn main() -> Result<()> { slog::debug!(state.log, "client response"; "result" => ?result); } }) - .with_patch("LinkId", &TypePatch::default() - .with_derive("Eq") - .with_derive("PartialEq") - ) - .with_patch("LinkCreate", &TypePatch::default() - .with_derive("Eq") - .with_derive("PartialEq") - ) - .with_patch("LinkSettings", &TypePatch::default() - .with_derive("Eq") - .with_derive("PartialEq") - ) - .with_patch("PortSettings", &TypePatch::default() - .with_derive("Eq") - .with_derive("PartialEq") - ) + .with_derive("PartialEq") ) .generate_tokens(&spec) .with_context(|| { diff --git a/clients/repo-depot-client/Cargo.toml b/clients/repo-depot-client/Cargo.toml new file mode 100644 index 0000000000..858c75632f --- /dev/null +++ b/clients/repo-depot-client/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "repo-depot-client" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[lints] +workspace = true + +[dependencies] +omicron-workspace-hack.workspace = true +progenitor.workspace = true +reqwest.workspace = true +schemars.workspace = true +serde.workspace = true +slog.workspace = true diff --git a/clients/repo-depot-client/src/lib.rs b/clients/repo-depot-client/src/lib.rs new file mode 100644 index 0000000000..69e21cdaf3 --- /dev/null +++ b/clients/repo-depot-client/src/lib.rs @@ -0,0 +1,24 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Interface for Sled Agent's Repo Depot to make API requests. + +progenitor::generate_api!( + spec = "../../openapi/repo-depot.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + derives = [schemars::JsonSchema], +); + +/// A type alias for errors returned by this crate. +pub type ClientError = crate::Error; diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index dfa89f4cc6..0398e15e96 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -31,6 +31,7 @@ progenitor::generate_api!( BgpConfig = { derives = [Eq, Hash] }, BgpPeerConfig = { derives = [Eq, Hash] }, LldpPortConfig = { derives = [Eq, Hash, PartialOrd, Ord] }, + TxEqConfig = { derives = [Eq, Hash] }, OmicronPhysicalDiskConfig = { derives = [Eq, Hash, PartialOrd, Ord] }, PortConfigV2 = { derives = [Eq, Hash] }, RouteConfig = { derives = [Eq, Hash] }, @@ -43,6 +44,7 @@ progenitor::generate_api!( replace = { Baseboard = nexus_sled_agent_shared::inventory::Baseboard, ByteCount = omicron_common::api::external::ByteCount, + DatasetsConfig = omicron_common::disk::DatasetsConfig, DatasetKind = omicron_common::api::internal::shared::DatasetKind, DiskIdentity = omicron_common::disk::DiskIdentity, DiskVariant = omicron_common::disk::DiskVariant, diff --git a/cockroach-admin/src/cockroach_cli.rs b/cockroach-admin/src/cockroach_cli.rs index 0aa0532a23..ab1d470fe0 100644 --- a/cockroach-admin/src/cockroach_cli.rs +++ b/cockroach-admin/src/cockroach_cli.rs @@ -137,7 +137,7 @@ impl CockroachCli { mod tests { use super::*; use cockroach_admin_types::NodeMembership; - use nexus_test_utils::db::test_setup_database; + use nexus_test_utils::db::TestDatabase; use omicron_test_utils::dev; use std::net::SocketAddr; use url::Url; @@ -179,8 +179,8 @@ mod tests { #[tokio::test] async fn test_node_status_compatibility() { let logctx = dev::test_setup_log("test_node_status_compatibility"); - let mut db = test_setup_database(&logctx.log).await; - let db_url = db.listen_url().to_string(); + let db = TestDatabase::new_populate_nothing(&logctx.log).await; + let db_url = db.crdb().listen_url().to_string(); let expected_headers = "id,address,sql_address,build,started_at,updated_at,locality,is_available,is_live"; @@ -225,7 +225,7 @@ mod tests { assert_eq!(status[0].is_available, true); assert_eq!(status[0].is_live, true); - db.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } @@ -236,8 +236,8 @@ mod tests { async fn test_node_decommission_compatibility() { let logctx = dev::test_setup_log("test_node_decommission_compatibility"); - let mut db = test_setup_database(&logctx.log).await; - let db_url = db.listen_url().to_string(); + let db = TestDatabase::new_populate_nothing(&logctx.log).await; + let db_url = db.crdb().listen_url().to_string(); let expected_headers = "id,is_live,replicas,is_decommissioning,membership,is_draining"; @@ -291,7 +291,7 @@ mod tests { assert_eq!(result.is_draining, false); assert_eq!(result.notes, &[] as &[&str]); - db.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } } diff --git a/cockroach-admin/src/context.rs b/cockroach-admin/src/context.rs index ea281f7b75..4ba387bb6b 100644 --- a/cockroach-admin/src/context.rs +++ b/cockroach-admin/src/context.rs @@ -138,7 +138,7 @@ impl ServerContext { #[cfg(test)] mod tests { use super::*; - use nexus_test_utils::db::test_setup_database; + use nexus_test_utils::db::TestDatabase; use omicron_test_utils::dev; use std::net::SocketAddrV6; use url::Url; @@ -146,10 +146,11 @@ mod tests { #[tokio::test] async fn test_node_id() { let logctx = dev::test_setup_log("test_node_id"); - let mut db = test_setup_database(&logctx.log).await; + let db = TestDatabase::new_populate_nothing(&logctx.log).await; + let crdb = db.crdb(); // Construct a `ServerContext`. - let db_url = db.listen_url().to_string(); + let db_url = crdb.listen_url().to_string(); let url: Url = db_url.parse().expect("valid url"); let cockroach_address: SocketAddrV6 = format!( "{}:{}", @@ -173,7 +174,7 @@ mod tests { // The `OnceCell` should be populated now; even if we shut down the DB, // we can still fetch the node ID. - db.cleanup().await.unwrap(); + db.terminate().await; let node_id = context.node_id().await.expect("successfully read node ID"); assert_eq!(node_id, "1"); diff --git a/common/src/address.rs b/common/src/address.rs index 7cf00d5228..7e6d68ebc8 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -29,6 +29,7 @@ pub const MIN_PORT: u16 = u16::MIN; pub const DNS_PORT: u16 = 53; pub const DNS_HTTP_PORT: u16 = 5353; pub const SLED_AGENT_PORT: u16 = 12345; +pub const REPO_DEPOT_PORT: u16 = 12348; pub const COCKROACH_PORT: u16 = 32221; pub const COCKROACH_ADMIN_PORT: u16 = 32222; diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 083497258c..18343ac44a 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1225,13 +1225,16 @@ pub struct InstanceAutoRestartStatus { #[serde(rename = "auto_restart_enabled")] pub enabled: bool, - /// The auto-restart policy configured for this instance, or `None` if no - /// explicit policy is configured. + /// The auto-restart policy configured for this instance, or `null` if no + /// explicit policy has been configured. /// - /// If this is not present, then this instance uses the default auto-restart - /// policy, which may or may not allow it to be restarted. The - /// `auto_restart_enabled` field indicates whether the instance will be - /// automatically restarted. + /// This policy determines whether the instance should be automatically + /// restarted by the control plane on failure. If this is `null`, the + /// control plane will use the default policy when determining whether or + /// not to automatically restart this instance, which may or may not allow + /// it to be restarted. The value of the `auto_restart_enabled` field + /// indicates whether the instance will be auto-restarted, based on its + /// current policy or the default if it has no configured policy. // // Rename this field, as the struct is `#[serde(flatten)]`ed into the // `Instance` type, and we would like the field to be prefixed with @@ -2355,6 +2358,10 @@ pub struct SwitchPortSettingsView { /// Link-layer discovery protocol (LLDP) settings. pub link_lldp: Vec, + /// TX equalization settings. These are optional, and most links will not + /// need them. + pub tx_eq: Vec>, + /// Layer 3 interface settings. pub interfaces: Vec, @@ -2497,6 +2504,9 @@ pub struct SwitchPortLinkConfig { /// link. pub lldp_link_config_id: Option, + /// The tx_eq configuration id for this link. + pub tx_eq_config_id: Option, + /// The name of this link. pub link_name: String, @@ -2541,6 +2551,34 @@ pub struct LldpLinkConfig { pub management_ip: Option, } +/// Per-port tx-eq overrides. This can be used to fine-tune the transceiver +/// equalization settings to improve signal integrity. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct TxEqConfig { + /// Pre-cursor tap1 + pub pre1: Option, + /// Pre-cursor tap2 + pub pre2: Option, + /// Main tap + pub main: Option, + /// Post-cursor tap2 + pub post2: Option, + /// Post-cursor tap1 + pub post1: Option, +} + +impl From for TxEqConfig { + fn from(x: crate::api::internal::shared::TxEqConfig) -> TxEqConfig { + TxEqConfig { + pre1: x.pre1, + pre2: x.pre2, + main: x.main, + post2: x.post2, + post1: x.post1, + } + } +} + /// Describes the kind of an switch interface. #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] #[serde(rename_all = "snake_case")] diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 7776958254..5b1a528b36 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -458,6 +458,24 @@ pub struct LldpPortConfig { pub management_addrs: Option>, } +/// Per-port tx-eq overrides. This can be used to fine-tune the transceiver +/// equalization settings to improve signal integrity. +#[derive( + Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, JsonSchema, +)] +pub struct TxEqConfig { + /// Pre-cursor tap1 + pub pre1: Option, + /// Pre-cursor tap2 + pub pre2: Option, + /// Main tap + pub main: Option, + /// Post-cursor tap2 + pub post2: Option, + /// Post-cursor tap1 + pub post1: Option, +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, JsonSchema)] pub struct PortConfigV2 { /// The set of routes associated with this port. @@ -479,6 +497,8 @@ pub struct PortConfigV2 { pub autoneg: bool, /// LLDP configuration for this port pub lldp: Option, + /// TX-EQ configuration for this port + pub tx_eq: Option, } /// A set of switch uplinks. @@ -497,11 +517,17 @@ pub struct HostPortConfig { pub addrs: Vec, pub lldp: Option, + pub tx_eq: Option, } impl From for HostPortConfig { fn from(x: PortConfigV2) -> Self { - Self { port: x.port, addrs: x.addresses, lldp: x.lldp.clone() } + Self { + port: x.port, + addrs: x.addresses, + lldp: x.lldp.clone(), + tx_eq: x.tx_eq, + } } } @@ -860,8 +886,8 @@ pub enum DatasetKind { InternalDns, // Zone filesystems - ZoneRoot, - Zone { + TransientZoneRoot, + TransientZone { name: String, }, @@ -929,7 +955,7 @@ impl DatasetKind { match self { Cockroach | Crucible | Clickhouse | ClickhouseKeeper | ClickhouseServer | ExternalDns | InternalDns => true, - ZoneRoot | Zone { .. } | Debug | Update => false, + TransientZoneRoot | TransientZone { .. } | Debug | Update => false, } } @@ -937,7 +963,7 @@ impl DatasetKind { /// /// Otherwise, returns "None". pub fn zone_name(&self) -> Option<&str> { - if let DatasetKind::Zone { name } = self { + if let DatasetKind::TransientZone { name } = self { Some(name) } else { None @@ -961,8 +987,8 @@ impl fmt::Display for DatasetKind { ClickhouseServer => "clickhouse_server", ExternalDns => "external_dns", InternalDns => "internal_dns", - ZoneRoot => "zone", - Zone { name } => { + TransientZoneRoot => "zone", + TransientZone { name } => { write!(f, "zone/{}", name)?; return Ok(()); } @@ -992,12 +1018,12 @@ impl FromStr for DatasetKind { "clickhouse_server" => ClickhouseServer, "external_dns" => ExternalDns, "internal_dns" => InternalDns, - "zone" => ZoneRoot, + "zone" => TransientZoneRoot, "debug" => Debug, "update" => Update, other => { if let Some(name) = other.strip_prefix("zone/") { - Zone { name: name.to_string() } + TransientZone { name: name.to_string() } } else { return Err(DatasetKindParseError::UnknownDataset( s.to_string(), @@ -1087,8 +1113,8 @@ mod tests { DatasetKind::ClickhouseServer, DatasetKind::ExternalDns, DatasetKind::InternalDns, - DatasetKind::ZoneRoot, - DatasetKind::Zone { name: String::from("myzone") }, + DatasetKind::TransientZoneRoot, + DatasetKind::TransientZone { name: String::from("myzone") }, DatasetKind::Debug, DatasetKind::Update, ]; diff --git a/common/src/disk.rs b/common/src/disk.rs index ac9232e257..79e5a46f23 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -12,6 +12,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fmt; +use std::str::FromStr; use uuid::Uuid; use crate::{ @@ -186,6 +187,18 @@ impl GzipLevel { } } +impl FromStr for GzipLevel { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let level = s.parse::()?; + if level < GZIP_LEVEL_MIN || level > GZIP_LEVEL_MAX { + bail!("Invalid gzip compression level: {level}"); + } + Ok(Self(level)) + } +} + #[derive( Copy, Clone, @@ -224,6 +237,7 @@ pub enum CompressionAlgorithm { Zle, } +/// These match the arguments which can be passed to "zfs set compression=..." impl fmt::Display for CompressionAlgorithm { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { use CompressionAlgorithm::*; @@ -242,6 +256,29 @@ impl fmt::Display for CompressionAlgorithm { } } +impl FromStr for CompressionAlgorithm { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + use CompressionAlgorithm::*; + let c = match s { + "on" => On, + "" | "off" => Off, + "gzip" => Gzip, + "lz4" => Lz4, + "lzjb" => Lzjb, + "zle" => Zle, + _ => { + let Some(suffix) = s.strip_prefix("gzip-") else { + bail!("Unknown compression algorithm {s}"); + }; + GzipN { level: suffix.parse()? } + } + }; + Ok(c) + } +} + /// Configuration information necessary to request a single dataset #[derive( Clone, diff --git a/dev-tools/ls-apis/api-manifest.toml b/dev-tools/ls-apis/api-manifest.toml index f91aba74aa..ab5dd4dec8 100644 --- a/dev-tools/ls-apis/api-manifest.toml +++ b/dev-tools/ls-apis/api-manifest.toml @@ -278,6 +278,11 @@ client_package_name = "sled-agent-client" label = "Sled Agent" server_package_name = "sled-agent-api" +[[apis]] +client_package_name = "repo-depot-client" +label = "Repo Depot API" +server_package_name = "repo-depot-api" + [[apis]] client_package_name = "wicketd-client" label = "Wicketd" @@ -413,6 +418,17 @@ note = """ nexus-types depends on gateway-client for defining some types. """ +[[dependency_filter_rules]] +ancestor = "internal-dns" +client = "dns-service-client" +evaluation = "bogus" +note = """ +Past versions of internal-dns (which does not exist any more) depended on +dns-service-client for defining some types. We can remove this when other repos +that depend on Omicron have updated past the removal of the "internal-dns" +package. +""" + [[dependency_filter_rules]] ancestor = "nexus-types" client = "dns-service-client" diff --git a/dev-tools/ls-apis/src/bin/ls-apis.rs b/dev-tools/ls-apis/src/bin/ls-apis.rs index 39d1865d5b..9d2a5b5349 100644 --- a/dev-tools/ls-apis/src/bin/ls-apis.rs +++ b/dev-tools/ls-apis/src/bin/ls-apis.rs @@ -136,14 +136,29 @@ fn run_apis(apis: &SystemApis, args: ShowDepsArgs) -> Result<()> { let metadata = apis.api_metadata(); for api in metadata.apis() { println!("{} (client: {})", api.label, api.client_package_name); - for (s, path) in + for (s, dep_paths) in apis.api_consumers(&api.client_package_name, args.filter)? { let (repo_name, package_path) = apis.package_label(s)?; - println!(" consumed by: {} ({}/{})", s, repo_name, package_path); + println!( + " consumed by: {} ({}/{}) via {} path{}", + s, + repo_name, + package_path, + dep_paths.len(), + if dep_paths.len() == 1 { "" } else { "s" }, + ); if args.show_deps { - for p in path.nodes() { - println!(" via {}", p); + for (i, dep_path) in dep_paths.iter().enumerate() { + let label = if dep_paths.len() > 1 { + format!(" path {}", i + 1) + } else { + String::new() + }; + + for p in dep_path.nodes() { + println!(" via{}: {}", label, p); + } } } } diff --git a/dev-tools/ls-apis/src/cargo.rs b/dev-tools/ls-apis/src/cargo.rs index 1a7e87dd0c..f3e5f56bc3 100644 --- a/dev-tools/ls-apis/src/cargo.rs +++ b/dev-tools/ls-apis/src/cargo.rs @@ -9,7 +9,7 @@ use anyhow::bail; use anyhow::{anyhow, ensure, Context, Result}; use camino::Utf8Path; use camino::Utf8PathBuf; -use cargo_metadata::Package; +use cargo_metadata::{CargoOpt, Package}; use cargo_metadata::{DependencyKind, PackageId}; use std::collections::BTreeSet; use std::collections::{BTreeMap, VecDeque}; @@ -50,6 +50,7 @@ impl Workspace { pub fn load( name: &str, manifest_path: Option<&Utf8Path>, + extra_features: Option, ignored_non_clients: &BTreeSet, ) -> Result { eprintln!( @@ -65,6 +66,9 @@ impl Workspace { if let Some(manifest_path) = manifest_path { cmd.manifest_path(manifest_path); } + if let Some(extra_features) = extra_features { + cmd.features(extra_features); + } let metadata = match cmd.exec() { Err(original_err) if name == "maghemite" => { dendrite_workaround(cmd, original_err)? @@ -268,11 +272,6 @@ impl Workspace { while let Some(Remaining { node: next, path }) = remaining.pop() { for d in &next.deps { let did = &d.pkg; - if seen.contains(did) { - continue; - } - - seen.insert(did.clone()); if !d.dep_kinds.iter().any(|k| { matches!( k.kind, @@ -290,8 +289,13 @@ impl Workspace { let dep_pkg = self.packages_by_id.get(did).unwrap(); let dep_node = self.nodes_by_id.get(did).unwrap(); func(dep_pkg, &path); + if seen.contains(did) { + continue; + } + + seen.insert(did.clone()); let dep_path = path.with_dependency_on(did.clone()); - remaining.push(Remaining { node: dep_node, path: dep_path }) + remaining.push(Remaining { node: dep_node, path: dep_path }); } } diff --git a/dev-tools/ls-apis/src/system_apis.rs b/dev-tools/ls-apis/src/system_apis.rs index 60b9b9246a..c67db1f182 100644 --- a/dev-tools/ls-apis/src/system_apis.rs +++ b/dev-tools/ls-apis/src/system_apis.rs @@ -244,7 +244,7 @@ impl SystemApis { &self, client: &ClientPackageName, filter: ApiDependencyFilter, - ) -> Result + '_> + ) -> Result)> + '_> { let mut rv = Vec::new(); @@ -253,7 +253,7 @@ impl SystemApis { }; for (server_pkgname, dep_paths) in api_consumers { - let mut include = None; + let mut include = Vec::new(); for p in dep_paths { if filter.should_include( &self.api_metadata, @@ -261,13 +261,12 @@ impl SystemApis { &client, p, )? { - include = Some(p); - break; + include.push(p); } } - if let Some(p) = include { - rv.push((server_pkgname, p)) + if !include.is_empty() { + rv.push((server_pkgname, include)) } } diff --git a/dev-tools/ls-apis/src/workspaces.rs b/dev-tools/ls-apis/src/workspaces.rs index ace565e011..54df7a44e3 100644 --- a/dev-tools/ls-apis/src/workspaces.rs +++ b/dev-tools/ls-apis/src/workspaces.rs @@ -9,6 +9,7 @@ use crate::cargo::Workspace; use crate::ClientPackageName; use anyhow::{anyhow, ensure, Context, Result}; use camino::Utf8Path; +use cargo_metadata::CargoOpt; use cargo_metadata::Package; use cargo_metadata::PackageId; use std::collections::BTreeMap; @@ -35,8 +36,12 @@ impl Workspaces { // First, load information about the "omicron" workspace. This is the // current workspace so we don't need to provide the path to it. let ignored_non_clients = api_metadata.ignored_non_clients(); - let omicron = - Arc::new(Workspace::load("omicron", None, ignored_non_clients)?); + let omicron = Arc::new(Workspace::load( + "omicron", + None, + None, + ignored_non_clients, + )?); // In order to assemble this metadata, Cargo already has a clone of most // of the other workspaces that we care about. We'll use those clones @@ -55,17 +60,36 @@ impl Workspaces { // concurrency. let handles: Vec<_> = [ // To find this repo ... look up this package in Omicron - // v v - ("crucible", "crucible-agent-client"), - ("propolis", "propolis-client"), - ("maghemite", "mg-admin-client"), + // | | +---- and enable these extra + // | | | features when loading + // v v v + ("crucible", "crucible-agent-client", None), + ( + "propolis", + "propolis-client", + // The artifacts shipped from the Propolis repo (particularly, + // `propolis-server`) are built with the `omicron-build` + // feature, which is not enabled by default. Enable this + // feature when loading the Propolis repo metadata so that we + // see the dependency tree that a shipping system will have. + Some(CargoOpt::SomeFeatures(vec![String::from( + "omicron-build", + )])), + ), + ("maghemite", "mg-admin-client", None), ] .into_iter() - .map(|(repo, omicron_pkg)| { + .map(|(repo, omicron_pkg, extra_features)| { let mine = omicron.clone(); let my_ignored = ignored_non_clients.clone(); std::thread::spawn(move || { - load_dependent_repo(&mine, repo, omicron_pkg, my_ignored) + load_dependent_repo( + &mine, + repo, + omicron_pkg, + extra_features, + my_ignored, + ) }) }) .collect(); @@ -97,6 +121,7 @@ impl Workspaces { &maghemite, "dendrite", "dpd-client", + None, ignored_non_clients.clone(), )?, ); @@ -231,6 +256,7 @@ fn load_dependent_repo( workspace: &Workspace, repo: &str, pkgname: &str, + extra_features: Option, ignored_non_clients: BTreeSet, ) -> Result { // `Workspace` doesn't let us look up a non-workspace package by name @@ -288,5 +314,10 @@ fn load_dependent_repo( ) })?; let workspace_manifest = Utf8Path::new(output.trim_end()); - Workspace::load(repo, Some(workspace_manifest), &ignored_non_clients) + Workspace::load( + repo, + Some(workspace_manifest), + extra_features, + &ignored_non_clients, + ) } diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index a902484a75..aee8cd7a70 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -1,80 +1,83 @@ Bootstrap Agent (client: bootstrap-agent-client) - consumed by: omicron-sled-agent (omicron/sled-agent) - consumed by: wicketd (omicron/wicketd) + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path + consumed by: wicketd (omicron/wicketd) via 2 paths Clickhouse Cluster Admin for Keepers (client: clickhouse-admin-keeper-client) - consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-nexus (omicron/nexus) via 3 paths Clickhouse Cluster Admin for Servers (client: clickhouse-admin-server-client) - consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-nexus (omicron/nexus) via 3 paths CockroachDB Cluster Admin (client: cockroach-admin-client) - consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-nexus (omicron/nexus) via 2 paths Crucible Agent (client: crucible-agent-client) - consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-nexus (omicron/nexus) via 1 path Crucible Control (for testing only) (client: crucible-control-client) Crucible Pantry (client: crucible-pantry-client) - consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-nexus (omicron/nexus) via 1 path Maghemite DDM Admin (client: ddm-admin-client) - consumed by: installinator (omicron/installinator) - consumed by: mgd (maghemite/mgd) - consumed by: omicron-sled-agent (omicron/sled-agent) - consumed by: wicketd (omicron/wicketd) + consumed by: installinator (omicron/installinator) via 1 path + consumed by: mgd (maghemite/mgd) via 1 path + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path + consumed by: wicketd (omicron/wicketd) via 1 path DNS Server (client: dns-service-client) - consumed by: omicron-nexus (omicron/nexus) - consumed by: omicron-sled-agent (omicron/sled-agent) + consumed by: omicron-nexus (omicron/nexus) via 1 path + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path Dendrite DPD (client: dpd-client) - consumed by: ddmd (maghemite/ddmd) - consumed by: mgd (maghemite/mgd) - consumed by: omicron-nexus (omicron/nexus) - consumed by: omicron-sled-agent (omicron/sled-agent) - consumed by: tfportd (dendrite/tfportd) - consumed by: wicketd (omicron/wicketd) + consumed by: ddmd (maghemite/ddmd) via 2 paths + consumed by: mgd (maghemite/mgd) via 1 path + consumed by: omicron-nexus (omicron/nexus) via 1 path + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path + consumed by: tfportd (dendrite/tfportd) via 1 path + consumed by: wicketd (omicron/wicketd) via 2 paths Downstairs Controller (debugging only) (client: dsc-client) Management Gateway Service (client: gateway-client) - consumed by: dpd (dendrite/dpd) - consumed by: omicron-nexus (omicron/nexus) - consumed by: omicron-sled-agent (omicron/sled-agent) - consumed by: wicketd (omicron/wicketd) + consumed by: dpd (dendrite/dpd) via 1 path + consumed by: omicron-nexus (omicron/nexus) via 3 paths + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path + consumed by: wicketd (omicron/wicketd) via 3 paths Wicketd Installinator (client: installinator-client) - consumed by: installinator (omicron/installinator) + consumed by: installinator (omicron/installinator) via 1 path Maghemite MG Admin (client: mg-admin-client) - consumed by: omicron-nexus (omicron/nexus) - consumed by: omicron-sled-agent (omicron/sled-agent) + consumed by: omicron-nexus (omicron/nexus) via 1 path + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path Nexus Internal API (client: nexus-client) - consumed by: dpd (dendrite/dpd) - consumed by: omicron-nexus (omicron/nexus) - consumed by: omicron-sled-agent (omicron/sled-agent) - consumed by: oximeter-collector (omicron/oximeter/collector) - consumed by: propolis-server (propolis/bin/propolis-server) + consumed by: dpd (dendrite/dpd) via 1 path + consumed by: omicron-nexus (omicron/nexus) via 1 path + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path + consumed by: oximeter-collector (omicron/oximeter/collector) via 1 path + consumed by: propolis-server (propolis/bin/propolis-server) via 3 paths External API (client: oxide-client) Oximeter (client: oximeter-client) - consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-nexus (omicron/nexus) via 2 paths Propolis (client: propolis-client) - consumed by: omicron-nexus (omicron/nexus) - consumed by: omicron-sled-agent (omicron/sled-agent) + consumed by: omicron-nexus (omicron/nexus) via 2 paths + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path Crucible Repair (client: repair-client) - consumed by: crucible-downstairs (crucible/downstairs) + consumed by: crucible-downstairs (crucible/downstairs) via 1 path + +Repo Depot API (client: repo-depot-client) + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path Sled Agent (client: sled-agent-client) - consumed by: dpd (dendrite/dpd) - consumed by: omicron-nexus (omicron/nexus) - consumed by: omicron-sled-agent (omicron/sled-agent) + consumed by: dpd (dendrite/dpd) via 1 path + consumed by: omicron-nexus (omicron/nexus) via 7 paths + consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path Wicketd (client: wicketd-client) diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 3a5281a3a8..d971545087 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -464,8 +464,20 @@ struct InstanceInfoArgs { /// /// note that this is not exhaustive, as some VMM or migration records may /// have been hard-deleted. + /// + /// this is also enabled by `--all`. #[arg(short = 'i', long)] history: bool, + + /// include virtual resources provisioned by this instance. + /// + /// this is also enabled by `--all`. + #[arg(short = 'r', long)] + resources: bool, + + /// include all optional output. + #[arg(short = 'a', long)] + all: bool, } #[derive(Debug, Args)] @@ -1287,10 +1299,10 @@ async fn lookup_project( #[derive(Tabled)] #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] struct DiskIdentity { - name: String, id: Uuid, size: String, state: String, + name: String, } impl From<&'_ db::model::Disk> for DiskIdentity { @@ -2990,7 +3002,7 @@ async fn cmd_db_instance_info( Instance, InstanceKarmicStatus, InstanceRuntimeState, Migration, Reincarnatability, }; - let &InstanceInfoArgs { ref id, history } = args; + let &InstanceInfoArgs { ref id, history, resources, all } = args; let instance = instance_dsl::instance .filter(instance_dsl::id.eq(id.into_untyped_uuid())) @@ -3149,10 +3161,14 @@ async fn cmd_db_instance_info( ); match can_reincarnate { Reincarnatability::WillReincarnate => { - println!(" {KARMIC_STATUS:>WIDTH$}: bound to saṃsāra"); + println!( + " {KARMIC_STATUS:>WIDTH$}: saṃsāra (reincarnation enabled)" + ); } Reincarnatability::Nirvana => { - println!(" {KARMIC_STATUS:>WIDTH$}: attained nirvāṇa"); + println!( + " {KARMIC_STATUS:>WIDTH$}: nirvāṇa (reincarnation disabled)" + ); } Reincarnatability::CoolingDown(remaining) => { println!( @@ -3306,24 +3322,72 @@ async fn cmd_db_instance_info( } if !disks.is_empty() { - println!("\n{:=<80}\n", "== ATTACHED DISKS "); + println!("\n{:=<80}", "== ATTACHED DISKS "); check_limit(&disks, fetch_opts.fetch_limit, ctx); - let table = if fetch_opts.include_deleted { + let mut table = if fetch_opts.include_deleted { tabled::Table::new(disks.iter().map(MaybeDeletedDiskRow::from)) - .with(tabled::settings::Style::empty()) - .with(tabled::settings::Padding::new(0, 1, 0, 0)) - .to_string() } else { tabled::Table::new(disks.iter().map(DiskRow::from)) - .with(tabled::settings::Style::empty()) - .with(tabled::settings::Padding::new(0, 1, 0, 0)) - .to_string() }; + table + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)); println!("{table}"); } - if history { + if resources || all { + use db::schema::virtual_provisioning_resource::dsl as resource_dsl; + let resources = resource_dsl::virtual_provisioning_resource + .filter(resource_dsl::id.eq(id.into_untyped_uuid())) + .select(db::model::VirtualProvisioningResource::as_select()) + .load_async(&*datastore.pool_connection_for_tests().await?) + .await + .context("fetching instance virtual provisioning record")?; + println!("\n{:=<80}", "== VIRTUAL RESOURCES PROVISIONED "); + if resources.is_empty() { + println!("(i) no virtual resources provisioned for this instance") + } else { + if resources.len() > 1 { + println!( + "/!\\ there should only be one virtual resource record \ + for a given UUID! this is a bug!", + ); + } + for resource in resources { + let db::model::VirtualProvisioningResource { + id: _, + time_modified, + resource_type, + virtual_disk_bytes_provisioned: db::model::ByteCount(disk), + cpus_provisioned, + ram_provisioned: db::model::ByteCount(ram), + } = resource; + const DISK: &'static str = "virtual disk"; + const RAM: &'static str = "RAM"; + const WIDTH: usize = crate::helpers::const_max_len(&[ + VCPUS, + DISK, + RAM, + LAST_UPDATED, + ]); + if resource_type != "instance" { + println!( + "/!\\ virtual provisioning resource type is \ + {resource_type:?} (expected \"instance\")", + ); + } + println!(" {VCPUS:>WIDTH$}: {cpus_provisioned}"); + println!(" {RAM:>WIDTH$}: {ram}"); + println!(" {DISK:>WIDTH$}: {disk}"); + if let Some(modified) = time_modified { + println!(" {LAST_UPDATED:>WIDTH$}: {modified}") + } + } + } + } + + if history || all { let ctx = || "listing migrations"; let past_migrations = migration_dsl::migration .filter(migration_dsl::instance_id.eq(id.into_untyped_uuid())) @@ -3381,7 +3445,7 @@ async fn cmd_db_instance_info( .with_context(ctx)?; if !vmms.is_empty() { - println!("\n{:=<80}\n", "== VMM HISTORY"); + println!("\n{:=<80}", "== VMM HISTORY "); check_limit(&vmms, fetch_opts.fetch_limit, ctx); @@ -3435,11 +3499,11 @@ struct VmmStateRow { #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] struct CustomerInstanceRow { id: String, - name: String, state: String, propolis_id: MaybePropolisId, sled_id: MaybeSledId, host_serial: String, + name: String, } /// Run `omdb db instances`: list data about customer VMs. diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index 8d67cba3c6..cb05bb575b 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -35,6 +35,7 @@ use nexus_client::types::SagaState; use nexus_client::types::SledSelector; use nexus_client::types::UninitializedSledId; use nexus_db_queries::db::lookup::LookupPath; +use nexus_db_queries::db::DataStore; use nexus_inventory::now_db_precision; use nexus_saga_recovery::LastPass; use nexus_types::deployment::Blueprint; @@ -64,6 +65,7 @@ use slog_error_chain::InlineErrorChain; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::str::FromStr; +use std::sync::Arc; use tabled::settings::object::Columns; use tabled::settings::Padding; use tabled::Tabled; @@ -2709,6 +2711,27 @@ async fn cmd_nexus_sled_expunge( args: &SledExpungeArgs, omdb: &Omdb, log: &slog::Logger, + destruction_token: DestructiveOperationToken, +) -> Result<(), anyhow::Error> { + let datastore = args.db_url_opts.connect(omdb, log).await?; + let result = cmd_nexus_sled_expunge_with_datastore( + &datastore, + client, + args, + log, + destruction_token, + ) + .await; + datastore.terminate().await; + result +} + +// `omdb nexus sleds expunge`, but borrowing a datastore +async fn cmd_nexus_sled_expunge_with_datastore( + datastore: &Arc, + client: &nexus_client::Client, + args: &SledExpungeArgs, + log: &slog::Logger, _destruction_token: DestructiveOperationToken, ) -> Result<(), anyhow::Error> { // This is an extremely dangerous and irreversible operation. We put a @@ -2720,7 +2743,6 @@ async fn cmd_nexus_sled_expunge( // most recent inventory collection use nexus_db_queries::context::OpContext; - let datastore = args.db_url_opts.connect(omdb, log).await?; let opctx = OpContext::for_tests(log.clone(), datastore.clone()); let opctx = &opctx; @@ -2800,11 +2822,30 @@ async fn cmd_nexus_sled_expunge_disk( args: &DiskExpungeArgs, omdb: &Omdb, log: &slog::Logger, + destruction_token: DestructiveOperationToken, +) -> Result<(), anyhow::Error> { + let datastore = args.db_url_opts.connect(omdb, log).await?; + let result = cmd_nexus_sled_expunge_disk_with_datastore( + &datastore, + client, + args, + log, + destruction_token, + ) + .await; + datastore.terminate().await; + result +} + +async fn cmd_nexus_sled_expunge_disk_with_datastore( + datastore: &Arc, + client: &nexus_client::Client, + args: &DiskExpungeArgs, + log: &slog::Logger, _destruction_token: DestructiveOperationToken, ) -> Result<(), anyhow::Error> { use nexus_db_queries::context::OpContext; - let datastore = args.db_url_opts.connect(omdb, log).await?; let opctx = OpContext::for_tests(log.clone(), datastore.clone()); let opctx = &opctx; diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 1e99dbd3a8..6974c0b36b 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -2,7 +2,7 @@ EXECUTING COMMAND: omdb ["db", "disks", "list"] termination: Exited(0) --------------------------------------------- stdout: -NAME ID SIZE STATE ATTACHED_TO +ID SIZE STATE NAME ATTACHED_TO --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable @@ -52,7 +52,7 @@ EXECUTING COMMAND: omdb ["db", "instances"] termination: Exited(0) --------------------------------------------- stdout: -ID NAME STATE PROPOLIS_ID SLED_ID HOST_SERIAL +ID STATE PROPOLIS_ID SLED_ID HOST_SERIAL NAME --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable diff --git a/dev-tools/openapi-manager/Cargo.toml b/dev-tools/openapi-manager/Cargo.toml index 211e134016..d32477caf3 100644 --- a/dev-tools/openapi-manager/Cargo.toml +++ b/dev-tools/openapi-manager/Cargo.toml @@ -12,9 +12,9 @@ anyhow.workspace = true atomicwrites.workspace = true bootstrap-agent-api.workspace = true camino.workspace = true +clap.workspace = true clickhouse-admin-api.workspace = true cockroach-admin-api.workspace = true -clap.workspace = true dns-server-api.workspace = true dropshot.workspace = true fs-err.workspace = true @@ -24,13 +24,14 @@ installinator-api.workspace = true nexus-external-api.workspace = true nexus-internal-api.workspace = true omicron-workspace-hack.workspace = true -openapiv3.workspace = true openapi-lint.workspace = true openapi-manager-types.workspace = true +openapiv3.workspace = true owo-colors.workspace = true oximeter-api.workspace = true +repo-depot-api.workspace = true serde_json.workspace = true -sled-agent-api.workspace = true similar.workspace = true +sled-agent-api.workspace = true supports-color.workspace = true wicketd-api.workspace = true diff --git a/dev-tools/openapi-manager/src/spec.rs b/dev-tools/openapi-manager/src/spec.rs index dafcebac05..ff55bbeff5 100644 --- a/dev-tools/openapi-manager/src/spec.rs +++ b/dev-tools/openapi-manager/src/spec.rs @@ -121,6 +121,15 @@ pub fn all_apis() -> Vec { filename: "oximeter.json", extra_validation: None, }, + ApiSpec { + title: "Oxide TUF Repo Depot API", + version: "0.0.1", + description: "API for fetching update artifacts", + boundary: ApiBoundary::Internal, + api_description: repo_depot_api::repo_depot_api_mod::stub_api_description, + filename: "repo-depot.json", + extra_validation: None, + }, ApiSpec { title: "Oxide Sled Agent API", version: "0.0.1", diff --git a/dev-tools/reconfigurator-cli/src/main.rs b/dev-tools/reconfigurator-cli/src/main.rs index c973fa5af2..c23ad8e4be 100644 --- a/dev-tools/reconfigurator-cli/src/main.rs +++ b/dev-tools/reconfigurator-cli/src/main.rs @@ -686,9 +686,12 @@ fn cmd_sled_show( swriteln!(s, "sled {}", sled_id); swriteln!(s, "subnet {}", sled_resources.subnet.net()); swriteln!(s, "zpools ({}):", sled_resources.zpools.len()); - for (zpool, disk) in &sled_resources.zpools { + for (zpool, (disk, datasets)) in &sled_resources.zpools { swriteln!(s, " {:?}", zpool); - swriteln!(s, " ↳ {:?}", disk); + swriteln!(s, " {:?}", disk); + for dataset in datasets { + swriteln!(s, " ↳ {:?}", dataset); + } } Ok(Some(s)) } @@ -840,7 +843,12 @@ fn cmd_blueprint_edit( .context("failed to add Nexus zone")?; assert_matches::assert_matches!( added, - EnsureMultiple::Changed { added: 1, removed: 0 } + EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 0 + } ); format!("added Nexus zone to sled {}", sled_id) } @@ -852,7 +860,12 @@ fn cmd_blueprint_edit( .context("failed to add CockroachDB zone")?; assert_matches::assert_matches!( added, - EnsureMultiple::Changed { added: 1, removed: 0 } + EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 0 + } ); format!("added CockroachDB zone to sled {}", sled_id) } diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout index 838695cd80..93e705c6af 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout @@ -40,25 +40,25 @@ sled 2eb69596-f081-4e2d-9425-9994926e0832 subnet fd00:1122:3344:102::/64 zpools (10): 088ed702-551e-453b-80d7-57700372a844 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-088ed702-551e-453b-80d7-57700372a844" }, disk_id: b2850ccb-4ac7-4034-aeab-b1cd582d407b (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-088ed702-551e-453b-80d7-57700372a844" }, disk_id: b2850ccb-4ac7-4034-aeab-b1cd582d407b (physical_disk), policy: InService, state: Active } 09e51697-abad-47c0-a193-eaf74bc5d3cd (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-09e51697-abad-47c0-a193-eaf74bc5d3cd" }, disk_id: c6d1fe0d-5226-4318-a55a-e86e20612277 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-09e51697-abad-47c0-a193-eaf74bc5d3cd" }, disk_id: c6d1fe0d-5226-4318-a55a-e86e20612277 (physical_disk), policy: InService, state: Active } 3a512d49-edbe-47f3-8d0b-6051bfdc4044 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-3a512d49-edbe-47f3-8d0b-6051bfdc4044" }, disk_id: 24510d37-20b1-4bdc-9ca7-c37fff39abb2 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-3a512d49-edbe-47f3-8d0b-6051bfdc4044" }, disk_id: 24510d37-20b1-4bdc-9ca7-c37fff39abb2 (physical_disk), policy: InService, state: Active } 40517680-aa77-413c-bcf4-b9041dcf6612 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-40517680-aa77-413c-bcf4-b9041dcf6612" }, disk_id: 30ed317f-1717-4df6-8c1c-69f9d438705e (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-40517680-aa77-413c-bcf4-b9041dcf6612" }, disk_id: 30ed317f-1717-4df6-8c1c-69f9d438705e (physical_disk), policy: InService, state: Active } 78d3cb96-9295-4644-bf78-2e32191c71f9 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-78d3cb96-9295-4644-bf78-2e32191c71f9" }, disk_id: 5ac39660-8149-48a2-a6df-aebb0f30352a (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-78d3cb96-9295-4644-bf78-2e32191c71f9" }, disk_id: 5ac39660-8149-48a2-a6df-aebb0f30352a (physical_disk), policy: InService, state: Active } 853595e7-77da-404e-bc35-aba77478d55c (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-853595e7-77da-404e-bc35-aba77478d55c" }, disk_id: 43083372-c7d0-4df3-ac4e-96c45cde28d9 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-853595e7-77da-404e-bc35-aba77478d55c" }, disk_id: 43083372-c7d0-4df3-ac4e-96c45cde28d9 (physical_disk), policy: InService, state: Active } 8926e0e7-65d9-4e2e-ac6d-f1298af81ef1 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8926e0e7-65d9-4e2e-ac6d-f1298af81ef1" }, disk_id: 13e65865-2a6e-41f7-aa18-6ef8dff59b4e (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8926e0e7-65d9-4e2e-ac6d-f1298af81ef1" }, disk_id: 13e65865-2a6e-41f7-aa18-6ef8dff59b4e (physical_disk), policy: InService, state: Active } 9c0b9151-17f3-4857-94cc-b5bfcd402326 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9c0b9151-17f3-4857-94cc-b5bfcd402326" }, disk_id: 40383e60-18f6-4423-94e7-7b91ce939b43 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9c0b9151-17f3-4857-94cc-b5bfcd402326" }, disk_id: 40383e60-18f6-4423-94e7-7b91ce939b43 (physical_disk), policy: InService, state: Active } d61354fa-48d2-47c6-90bf-546e3ed1708b (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d61354fa-48d2-47c6-90bf-546e3ed1708b" }, disk_id: e02ae523-7b66-4188-93c8-c5808c01c795 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d61354fa-48d2-47c6-90bf-546e3ed1708b" }, disk_id: e02ae523-7b66-4188-93c8-c5808c01c795 (physical_disk), policy: InService, state: Active } d792c8cb-7490-40cb-bb1c-d4917242edf4 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d792c8cb-7490-40cb-bb1c-d4917242edf4" }, disk_id: c19e5610-a3a2-4cc6-af4d-517a49ef610b (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d792c8cb-7490-40cb-bb1c-d4917242edf4" }, disk_id: c19e5610-a3a2-4cc6-af4d-517a49ef610b (physical_disk), policy: InService, state: Active } > blueprint-show ade5749d-bdf3-4fab-a8ae-00bea01b3a5a @@ -494,13 +494,13 @@ sled 89d02b1b-478c-401a-8e28-7a26f74fa41b subnet fd00:1122:3344:101::/64 zpools (4): 44fa7024-c2bc-4d2c-b478-c4997e4aece8 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-44fa7024-c2bc-4d2c-b478-c4997e4aece8" }, disk_id: 2a15b33c-dd0e-45b7-aba9-d05f40f030ff (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-44fa7024-c2bc-4d2c-b478-c4997e4aece8" }, disk_id: 2a15b33c-dd0e-45b7-aba9-d05f40f030ff (physical_disk), policy: InService, state: Active } 8562317c-4736-4cfc-9292-7dcab96a6fee (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8562317c-4736-4cfc-9292-7dcab96a6fee" }, disk_id: cad6faa6-9409-4496-9aeb-392b3c50bed4 (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8562317c-4736-4cfc-9292-7dcab96a6fee" }, disk_id: cad6faa6-9409-4496-9aeb-392b3c50bed4 (physical_disk), policy: InService, state: Active } ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6 (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6" }, disk_id: 7d89a66e-0dcd-47ab-824d-62186812b8bd (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6" }, disk_id: 7d89a66e-0dcd-47ab-824d-62186812b8bd (physical_disk), policy: InService, state: Active } f931ec80-a3e3-4adb-a8ba-fa5adbd2294c (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-f931ec80-a3e3-4adb-a8ba-fa5adbd2294c" }, disk_id: 41755be9-2c77-4deb-87a4-cb53f09263fa (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-f931ec80-a3e3-4adb-a8ba-fa5adbd2294c" }, disk_id: 41755be9-2c77-4deb-87a4-cb53f09263fa (physical_disk), policy: InService, state: Active } > blueprint-show ade5749d-bdf3-4fab-a8ae-00bea01b3a5a diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-stdout index dc1ba0c1ca..40489caeb5 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmd-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmd-stdout @@ -24,25 +24,25 @@ sled ..................... subnet fd00:1122:3344:101::/64 zpools (10): ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } > sled-add ..................... @@ -98,24 +98,24 @@ sled ..................... subnet fd00:1122:3344:101::/64 zpools (10): ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } ..................... (zpool) - ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } + SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-....................." }, disk_id: ..................... (physical_disk), policy: InService, state: Active } diff --git a/dns-server/src/dns_server.rs b/dns-server/src/dns_server.rs index 34750719c1..6259c4951e 100644 --- a/dns-server/src/dns_server.rs +++ b/dns-server/src/dns_server.rs @@ -274,6 +274,19 @@ async fn handle_dns_message( let mut additional_records = vec![]; let response_records = records .into_iter() + .filter(|record| { + let ty = query.query_type(); + if ty == RecordType::ANY { + return true; + } + + match (ty, record) { + (RecordType::A, DnsRecord::A(_)) => true, + (RecordType::AAAA, DnsRecord::Aaaa(_)) => true, + (RecordType::SRV, DnsRecord::Srv(_)) => true, + _ => false, + } + }) .map(|record| { let record = dns_record_to_record(&name, record)?; diff --git a/dns-server/tests/basic_test.rs b/dns-server/tests/basic_test.rs index b555b82a80..30026dc08c 100644 --- a/dns-server/tests/basic_test.rs +++ b/dns-server/tests/basic_test.rs @@ -6,11 +6,20 @@ use anyhow::{Context, Result}; use camino_tempfile::Utf8TempDir; use dns_service_client::Client; use dropshot::{test_util::LogContext, HandlerTaskMode}; +use hickory_client::{ + client::{AsyncClient, ClientHandle}, + error::ClientError, + udp::UdpClientStream, +}; use hickory_resolver::error::ResolveErrorKind; use hickory_resolver::TokioAsyncResolver; use hickory_resolver::{ config::{NameServerConfig, Protocol, ResolverConfig, ResolverOpts}, - proto::op::ResponseCode, + proto::{ + op::ResponseCode, + rr::{DNSClass, Name, RecordType}, + xfer::DnsResponse, + }, }; use internal_dns_types::config::{ DnsConfigParams, DnsConfigZone, DnsRecord, Srv, @@ -85,6 +94,59 @@ pub async fn aaaa_crud() -> Result<(), anyhow::Error> { Ok(()) } +#[tokio::test] +pub async fn answers_match_question() -> Result<(), anyhow::Error> { + let test_ctx = init_client_server("answers_match_question").await?; + let client = &test_ctx.client; + + // records should initially be empty + let records = dns_records_list(client, TEST_ZONE).await?; + assert!(records.is_empty()); + + // add an aaaa record + let name = "devron".to_string(); + let addr = Ipv6Addr::new(0xfd, 0, 0, 0, 0, 0, 0, 0x1); + let aaaa = DnsRecord::Aaaa(addr); + let input_records = HashMap::from([(name.clone(), vec![aaaa])]); + dns_records_create(client, TEST_ZONE, input_records.clone()).await?; + + // read back the aaaa record + let records = dns_records_list(client, TEST_ZONE).await?; + assert_eq!(input_records, records); + + let name = Name::from_ascii(&(name + "." + TEST_ZONE + ".")) + .expect("can construct name for query"); + + // If a server returns answers incorrectly, such as sending AAAA answers to + // an A query, it turns out `hickory-resolver`'s internal CachingClient + // transparently corrects the misbehavior. The caching client will cache the + // extra record, then see there are no A records matching the query, and + // finally send a correct response with no answers. + // + // `raw_dns_client_query` avoids using a hickory Resolver, so we can assert + // on the exact answer from our server. + let response = raw_dns_client_query( + test_ctx.dns_server.local_address(), + name, + RecordType::A, + ) + .await + .expect("test query is ok"); + + // The answer we expect is: + // * no error: the domain exists, so NXDOMAIN would be wrong + // * no answers: we ask specifically for a record type the server does not + // have + // * no additionals: the server could return AAAA records as additionals to + // an A query, but does not currently. + assert_eq!(response.header().response_code(), ResponseCode::NoError); + assert_eq!(response.answers(), &[]); + assert_eq!(response.additionals(), &[]); + + test_ctx.cleanup().await; + Ok(()) +} + #[tokio::test] pub async fn srv_crud() -> Result<(), anyhow::Error> { let test_ctx = init_client_server("srv_crud").await?; @@ -97,6 +159,7 @@ pub async fn srv_crud() -> Result<(), anyhow::Error> { // add a srv record let name = "hromi".to_string(); + let test_fqdn = name.clone() + "." + TEST_ZONE + "."; let target = "outpost47"; let srv = Srv { prio: 47, @@ -121,8 +184,13 @@ pub async fn srv_crud() -> Result<(), anyhow::Error> { )]); dns_records_create(client, TEST_ZONE, input_records.clone()).await?; - // resolve the srv - let response = resolver.srv_lookup(name + "." + TEST_ZONE + ".").await?; + // resolve the srv. we'll test this in two ways: + // * the srv record as seen through `hickory_resolver`, the higher-level + // interface we use in many places + // * the srv record as seen through `hickory_client`, to double-check that + // the exact DNS response has the answers/additionals sections as we'd + // expect it to be + let response = resolver.srv_lookup(&test_fqdn).await?; let srvr = response.iter().next().expect("no srv records returned!"); assert_eq!(srvr.priority(), srv.prio); assert_eq!(srvr.weight(), srv.weight); @@ -132,6 +200,27 @@ pub async fn srv_crud() -> Result<(), anyhow::Error> { aaaa_records.sort(); assert_eq!(aaaa_records, [IpAddr::from(addr1), IpAddr::from(addr2)]); + // OK, through `hickory_resolver` everything looks right. now double-check + // that the additional records really do come back in the "Additionals" + // section of the response. + + let name = hickory_client::rr::domain::Name::from_ascii(&test_fqdn) + .expect("can construct name for query"); + + let response = raw_dns_client_query( + test_ctx.dns_server.local_address(), + name, + RecordType::SRV, + ) + .await + .expect("test query is ok"); + assert_eq!(response.header().response_code(), ResponseCode::NoError); + assert_eq!(response.answers().len(), 1); + assert_eq!(response.answers()[0].record_type(), RecordType::SRV); + assert_eq!(response.additionals().len(), 2); + assert_eq!(response.additionals()[0].record_type(), RecordType::AAAA); + assert_eq!(response.additionals()[1].record_type(), RecordType::AAAA); + test_ctx.cleanup().await; Ok(()) } @@ -481,3 +570,24 @@ async fn dns_records_list( .map(|z| z.records) .unwrap_or_else(HashMap::new)) } + +/// Issue a DNS query of `record_ty` records for `name`. +/// +/// In most tests we just use `hickory-resolver` for a higher-level interface to +/// making DNS queries and handling responses. This is also the crate used +/// elsewhere to handle DNS responses in Omicron. However, it is slightly +/// higher-level than the actual wire response we produce from the DNS server in +/// this same crate; to assert on responses we send on the wire, this issues a +/// query using `hickory-client` and returns the corresponding `DnsResponse`. +async fn raw_dns_client_query( + resolver_addr: std::net::SocketAddr, + name: Name, + record_ty: RecordType, +) -> Result { + let stream = UdpClientStream::::new(resolver_addr); + let (mut trust_client, bg) = AsyncClient::connect(stream).await.unwrap(); + + tokio::spawn(bg); + + trust_client.query(name, DNSClass::IN, record_ty).await +} diff --git a/flake.lock b/flake.lock index 2c0393f722..c2655cbe43 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1723175592, - "narHash": "sha256-M0xJ3FbDUc4fRZ84dPGx5VvgFsOzds77KiBMW/mMTnI=", + "lastModified": 1730200266, + "narHash": "sha256-l253w0XMT8nWHGXuXqyiIC/bMvh1VRszGXgdpQlfhvU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5e0ca22929f3342b19569b21b2f3462f053e497b", + "rev": "807e9154dcb16384b1b765ebe9cd2bba2ac287fd", "type": "github" }, "original": { @@ -29,11 +29,11 @@ ] }, "locked": { - "lastModified": 1723429325, - "narHash": "sha256-4x/32xTCd+xCwFoI/kKSiCr5LQA2ZlyTRYXKEni5HR8=", + "lastModified": 1730428392, + "narHash": "sha256-2aRfq1P0usr+TlW9LUCoefqqpPum873ac0TgZzXYHKI=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "65e3dc0fe079fe8df087cd38f1fe6836a0373aad", + "rev": "17eda17f5596a84e92ba94160139eb70f3c3e734", "type": "github" }, "original": { diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index 5dbe4338cf..1dda130c95 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -12,12 +12,13 @@ use crate::dladm::Etherstub; use crate::link::{Link, VnicAllocator}; use crate::opte::{Port, PortTicket}; use crate::svc::wait_for_service; -use crate::zone::{AddressRequest, ZONE_PREFIX}; +use crate::zone::AddressRequest; use crate::zpool::{PathInPool, ZpoolName}; use camino::{Utf8Path, Utf8PathBuf}; use camino_tempfile::Utf8TempDir; use ipnetwork::IpNetwork; use omicron_common::backoff; +use omicron_uuid_kinds::OmicronZoneUuid; pub use oxlog::is_oxide_smf_log_file; use slog::{error, info, o, warn, Logger}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; @@ -26,7 +27,6 @@ use std::sync::Arc; use std::sync::OnceLock; #[cfg(target_os = "illumos")] use std::thread; -use uuid::Uuid; #[cfg(any(test, feature = "testing"))] use crate::zone::MockZones as Zones; @@ -947,12 +947,11 @@ impl InstalledZone { /// /// This results in a zone name which is distinct across different zpools, /// but stable and predictable across reboots. - pub fn get_zone_name(zone_type: &str, unique_name: Option) -> String { - let mut zone_name = format!("{}{}", ZONE_PREFIX, zone_type); - if let Some(suffix) = unique_name { - zone_name.push_str(&format!("_{}", suffix)); - } - zone_name + pub fn get_zone_name( + zone_type: &str, + unique_name: Option, + ) -> String { + crate::zone::zone_name(zone_type, unique_name) } /// Get the name of the bootstrap VNIC in the zone, if any. @@ -1055,7 +1054,7 @@ pub struct ZoneBuilder<'a> { // builder purposes - that is, skipping this field in the builder will // still result in an `Ok(InstalledZone)` from `.install()`, rather than // an `Err(InstallZoneError::IncompleteBuilder)`. - unique_name: Option, + unique_name: Option, /// ZFS datasets to be accessed from within the zone. datasets: Option<&'a [zone::Dataset]>, /// Filesystems to mount within the zone. @@ -1119,7 +1118,7 @@ impl<'a> ZoneBuilder<'a> { } /// Unique ID of the instance of the zone being created. (optional) - pub fn with_unique_name(mut self, uuid: Uuid) -> Self { + pub fn with_unique_name(mut self, uuid: OmicronZoneUuid) -> Self { self.unique_name = Some(uuid); self } diff --git a/illumos-utils/src/zfs.rs b/illumos-utils/src/zfs.rs index f92fd5d60f..fa09fb22c5 100644 --- a/illumos-utils/src/zfs.rs +++ b/illumos-utils/src/zfs.rs @@ -289,6 +289,28 @@ impl FromStr for DatasetProperties { } } +#[derive(Debug, Copy, Clone)] +pub enum PropertySource { + Local, + Default, + Inherited, + Temporary, + None, +} + +impl fmt::Display for PropertySource { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let ps = match self { + PropertySource::Local => "local", + PropertySource::Default => "default", + PropertySource::Inherited => "inherited", + PropertySource::Temporary => "temporary", + PropertySource::None => "none", + }; + write!(f, "{ps}") + } +} + #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))] impl Zfs { /// Lists all datasets within a pool or existing dataset. @@ -646,7 +668,13 @@ impl Zfs { filesystem_name: &str, name: &str, ) -> Result { - Zfs::get_value(filesystem_name, &format!("oxide:{}", name)) + let property = format!("oxide:{name}"); + let [value] = Self::get_values( + filesystem_name, + &[&property], + Some(PropertySource::Local), + )?; + Ok(value) } /// Calls "zfs get" with a single value @@ -654,7 +682,7 @@ impl Zfs { filesystem_name: &str, name: &str, ) -> Result { - let [value] = Self::get_values(filesystem_name, &[name])?; + let [value] = Self::get_values(filesystem_name, &[name], None)?; Ok(value) } @@ -722,14 +750,24 @@ impl Zfs { // These methods don't work with mockall, so they exist in a separate impl block impl Zfs { /// Calls "zfs get" to acquire multiple values + /// + /// - `names`: The properties being acquired + /// - `source`: The optioanl property source (origin of the property) + /// Defaults to "all sources" when unspecified. pub fn get_values( filesystem_name: &str, names: &[&str; N], + source: Option, ) -> Result<[String; N], GetValueError> { let mut cmd = std::process::Command::new(PFEXEC); let all_names = names.into_iter().map(|n| *n).collect::>().join(","); - cmd.args(&[ZFS, "get", "-Ho", "value", &all_names, filesystem_name]); + + cmd.args(&[ZFS, "get", "-Ho", "value", &all_names]); + if let Some(source) = source { + cmd.args(&["-s", &source.to_string()]); + } + cmd.arg(filesystem_name); let output = execute(&mut cmd).map_err(|err| GetValueError { filesystem: filesystem_name.to_string(), name: format!("{:?}", names), diff --git a/illumos-utils/src/zone.rs b/illumos-utils/src/zone.rs index 47cc84dce6..da08c7b7df 100644 --- a/illumos-utils/src/zone.rs +++ b/illumos-utils/src/zone.rs @@ -17,6 +17,7 @@ use crate::dladm::{EtherstubVnic, VNIC_PREFIX_BOOTSTRAP, VNIC_PREFIX_CONTROL}; use crate::zpool::PathInPool; use crate::{execute, PFEXEC}; use omicron_common::address::SLED_PREFIX; +use omicron_uuid_kinds::OmicronZoneUuid; const DLADM: &str = "/usr/sbin/dladm"; pub const IPADM: &str = "/usr/sbin/ipadm"; @@ -29,6 +30,14 @@ pub const ROUTE: &str = "/usr/sbin/route"; pub const ZONE_PREFIX: &str = "oxz_"; pub const PROPOLIS_ZONE_PREFIX: &str = "oxz_propolis-server_"; +pub fn zone_name(prefix: &str, id: Option) -> String { + if let Some(id) = id { + format!("{ZONE_PREFIX}{}_{}", prefix, id) + } else { + format!("{ZONE_PREFIX}{}", prefix) + } +} + #[derive(thiserror::Error, Debug)] enum Error { #[error("Zone execution error: {0}")] diff --git a/live-tests/tests/test_nexus_add_remove.rs b/live-tests/tests/test_nexus_add_remove.rs index 6a0cdd93d0..81586496fd 100644 --- a/live-tests/tests/test_nexus_add_remove.rs +++ b/live-tests/tests/test_nexus_add_remove.rs @@ -70,7 +70,12 @@ async fn test_nexus_add_remove(lc: &LiveTestContext) { .context("adding Nexus zone")?; assert_matches!( count, - EnsureMultiple::Changed { added: 1, removed: 0 } + EnsureMultiple::Changed { + added: 1, + removed: 0, + updated: 0, + expunged: 0 + } ); Ok(()) }, diff --git a/nexus/benches/setup_benchmark.rs b/nexus/benches/setup_benchmark.rs index e882bf860c..e8fe4ab747 100644 --- a/nexus/benches/setup_benchmark.rs +++ b/nexus/benches/setup_benchmark.rs @@ -6,7 +6,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; use dropshot::test_util::LogContext; -use nexus_test_utils::db::test_setup_database; +use nexus_db_queries::db::pub_test_utils::TestDatabase; use omicron_test_utils::dev; // This is the default wrapper around most Nexus integration tests. @@ -22,8 +22,8 @@ async fn do_full_setup() { async fn do_crdb_setup() { let cfg = nexus_test_utils::load_test_config(); let logctx = LogContext::new("crdb_setup", &cfg.pkg.log); - let mut db = test_setup_database(&logctx.log).await; - db.cleanup().await.unwrap(); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + db.terminate().await; } // Wraps exclusively the ClickhouseDB portion of setup/teardown. diff --git a/nexus/db-model/src/dataset.rs b/nexus/db-model/src/dataset.rs index f896f11c5b..ad351fe612 100644 --- a/nexus/db-model/src/dataset.rs +++ b/nexus/db-model/src/dataset.rs @@ -2,13 +2,18 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use super::{DatasetKind, Generation, Region, SqlU16}; +use super::{ByteCount, DatasetKind, Generation, Region, SqlU16}; use crate::collection::DatastoreCollectionConfig; use crate::ipv6; use crate::schema::{dataset, region}; use chrono::{DateTime, Utc}; use db_macros::Asset; +use nexus_types::deployment::BlueprintDatasetConfig; +use omicron_common::api::external::Error; use omicron_common::api::internal::shared::DatasetKind as ApiDatasetKind; +use omicron_uuid_kinds::DatasetUuid; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::ZpoolUuid; use serde::{Deserialize, Serialize}; use std::net::{Ipv6Addr, SocketAddrV6}; use uuid::Uuid; @@ -43,6 +48,16 @@ pub struct Dataset { pub kind: DatasetKind, pub size_used: Option, zone_name: Option, + + quota: Option, + reservation: Option, + // This is the stringified form of + // "omicron_common::disk::CompressionAlgorithm". + // + // It can't serialize to the database without forcing omicron_common to + // depend on Diesel -- we could create a newtype, but "to_string" and + // "parse" cover this usage similarly. + compression: Option, } impl Dataset { @@ -55,7 +70,7 @@ impl Dataset { let kind = DatasetKind::from(&api_kind); let (size_used, zone_name) = match api_kind { ApiDatasetKind::Crucible => (Some(0), None), - ApiDatasetKind::Zone { name } => (None, Some(name)), + ApiDatasetKind::TransientZone { name } => (None, Some(name)), _ => (None, None), }; @@ -69,6 +84,9 @@ impl Dataset { kind, size_used, zone_name, + quota: None, + reservation: None, + compression: None, } } @@ -81,6 +99,69 @@ impl Dataset { } } +impl From for Dataset { + fn from(bp: BlueprintDatasetConfig) -> Self { + let kind = DatasetKind::from(&bp.kind); + let zone_name = bp.kind.zone_name().map(|s| s.to_string()); + // Only Crucible uses this "size_used" field. + let size_used = match bp.kind { + ApiDatasetKind::Crucible => Some(0), + ApiDatasetKind::Cockroach + | ApiDatasetKind::Clickhouse + | ApiDatasetKind::ClickhouseKeeper + | ApiDatasetKind::ClickhouseServer + | ApiDatasetKind::ExternalDns + | ApiDatasetKind::InternalDns + | ApiDatasetKind::TransientZone { .. } + | ApiDatasetKind::TransientZoneRoot + | ApiDatasetKind::Debug + | ApiDatasetKind::Update => None, + }; + let addr = bp.address; + Self { + identity: DatasetIdentity::new(bp.id.into_untyped_uuid()), + time_deleted: None, + rcgen: Generation::new(), + pool_id: bp.pool.id().into_untyped_uuid(), + kind, + ip: addr.map(|addr| addr.ip().into()), + port: addr.map(|addr| addr.port().into()), + size_used, + zone_name, + quota: bp.quota.map(ByteCount::from), + reservation: bp.reservation.map(ByteCount::from), + compression: Some(bp.compression.to_string()), + } + } +} + +impl TryFrom for omicron_common::disk::DatasetConfig { + type Error = Error; + + fn try_from(dataset: Dataset) -> Result { + let compression = if let Some(c) = dataset.compression { + c.parse().map_err(|e: anyhow::Error| { + Error::internal_error(&e.to_string()) + })? + } else { + omicron_common::disk::CompressionAlgorithm::Off + }; + + Ok(Self { + id: DatasetUuid::from_untyped_uuid(dataset.identity.id), + name: omicron_common::disk::DatasetName::new( + omicron_common::zpool_name::ZpoolName::new_external( + ZpoolUuid::from_untyped_uuid(dataset.pool_id), + ), + dataset.kind.try_into_api(dataset.zone_name)?, + ), + quota: dataset.quota.map(|q| q.into()), + reservation: dataset.reservation.map(|r| r.into()), + compression, + }) + } +} + // Datasets contain regions impl DatastoreCollectionConfig for Dataset { type CollectionId = Uuid; diff --git a/nexus/db-model/src/dataset_kind.rs b/nexus/db-model/src/dataset_kind.rs index 3f0b7c39bb..e90f0d1db3 100644 --- a/nexus/db-model/src/dataset_kind.rs +++ b/nexus/db-model/src/dataset_kind.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::impl_enum_type; +use omicron_common::api::external::Error; use omicron_common::api::internal; use serde::{Deserialize, Serialize}; @@ -23,12 +24,44 @@ impl_enum_type!( ClickhouseServer => b"clickhouse_server" ExternalDns => b"external_dns" InternalDns => b"internal_dns" - ZoneRoot => b"zone_root" - Zone => b"zone" + TransientZoneRoot => b"zone_root" + TransientZone => b"zone" Debug => b"debug" Update => b"update" ); +impl DatasetKind { + pub fn try_into_api( + self, + zone_name: Option, + ) -> Result { + use internal::shared::DatasetKind as ApiKind; + let k = match (self, zone_name) { + (Self::Crucible, None) => ApiKind::Crucible, + (Self::Cockroach, None) => ApiKind::Cockroach, + (Self::Clickhouse, None) => ApiKind::Clickhouse, + (Self::ClickhouseKeeper, None) => ApiKind::ClickhouseKeeper, + (Self::ClickhouseServer, None) => ApiKind::ClickhouseServer, + (Self::ExternalDns, None) => ApiKind::ExternalDns, + (Self::InternalDns, None) => ApiKind::InternalDns, + (Self::TransientZoneRoot, None) => ApiKind::TransientZoneRoot, + (Self::TransientZone, Some(name)) => { + ApiKind::TransientZone { name } + } + (Self::Debug, None) => ApiKind::Debug, + (Self::Update, None) => ApiKind::Update, + (Self::TransientZone, None) => { + return Err(Error::internal_error("Zone kind needs name")) + } + (_, Some(_)) => { + return Err(Error::internal_error("Only zone kind needs name")) + } + }; + + Ok(k) + } +} + impl From<&internal::shared::DatasetKind> for DatasetKind { fn from(k: &internal::shared::DatasetKind) -> Self { match k { @@ -49,12 +82,16 @@ impl From<&internal::shared::DatasetKind> for DatasetKind { internal::shared::DatasetKind::InternalDns => { DatasetKind::InternalDns } - internal::shared::DatasetKind::ZoneRoot => DatasetKind::ZoneRoot, + internal::shared::DatasetKind::TransientZoneRoot => { + DatasetKind::TransientZoneRoot + } // Enums in the database do not have associated data, so this drops // the "name" of the zone and only considers the type. // // The zone name, if it exists, is stored in a separate column. - internal::shared::DatasetKind::Zone { .. } => DatasetKind::Zone, + internal::shared::DatasetKind::TransientZone { .. } => { + DatasetKind::TransientZone + } internal::shared::DatasetKind::Debug => DatasetKind::Debug, internal::shared::DatasetKind::Update => DatasetKind::Update, } diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs index 693bec334d..fff829f548 100644 --- a/nexus/db-model/src/deployment.rs +++ b/nexus/db-model/src/deployment.rs @@ -10,29 +10,34 @@ use crate::omicron_zone_config::{self, OmicronZoneNic}; use crate::schema::{ blueprint, bp_clickhouse_cluster_config, bp_clickhouse_keeper_zone_id_to_node_id, - bp_clickhouse_server_zone_id_to_node_id, bp_omicron_physical_disk, - bp_omicron_zone, bp_omicron_zone_nic, bp_sled_omicron_physical_disks, + bp_clickhouse_server_zone_id_to_node_id, bp_omicron_dataset, + bp_omicron_physical_disk, bp_omicron_zone, bp_omicron_zone_nic, + bp_sled_omicron_datasets, bp_sled_omicron_physical_disks, bp_sled_omicron_zones, bp_sled_state, bp_target, }; use crate::typed_uuid::DbTypedUuid; use crate::{ - impl_enum_type, ipv6, Generation, MacAddr, Name, SledState, SqlU16, SqlU32, - SqlU8, + impl_enum_type, ipv6, ByteCount, Generation, MacAddr, Name, SledState, + SqlU16, SqlU32, SqlU8, }; use anyhow::{anyhow, bail, Context, Result}; use chrono::{DateTime, Utc}; use clickhouse_admin_types::{KeeperId, ServerId}; use ipnetwork::IpNetwork; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; +use nexus_types::deployment::blueprint_zone_type; +use nexus_types::deployment::BlueprintDatasetConfig; +use nexus_types::deployment::BlueprintDatasetDisposition; +use nexus_types::deployment::BlueprintDatasetsConfig; +use nexus_types::deployment::BlueprintPhysicalDiskConfig; +use nexus_types::deployment::BlueprintPhysicalDisksConfig; use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; +use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::BlueprintZonesConfig; +use nexus_types::deployment::ClickhouseClusterConfig; use nexus_types::deployment::CockroachDbPreserveDowngrade; -use nexus_types::deployment::{ - blueprint_zone_type, BlueprintPhysicalDisksConfig, ClickhouseClusterConfig, -}; -use nexus_types::deployment::{BlueprintPhysicalDiskConfig, BlueprintZoneType}; use nexus_types::deployment::{ OmicronZoneExternalFloatingAddr, OmicronZoneExternalFloatingIp, OmicronZoneExternalSnatIp, @@ -41,7 +46,7 @@ use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::disk::DiskIdentity; use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::{ - ExternalIpKind, ExternalIpUuid, GenericUuid, OmicronZoneKind, + DatasetKind, ExternalIpKind, ExternalIpUuid, GenericUuid, OmicronZoneKind, OmicronZoneUuid, SledKind, SledUuid, ZpoolKind, ZpoolUuid, }; use std::net::{IpAddr, SocketAddrV6}; @@ -210,6 +215,154 @@ impl From for BlueprintPhysicalDiskConfig { } } +impl_enum_type!( + #[derive(Clone, SqlType, Debug, QueryId)] + #[diesel(postgres_type(name = "bp_dataset_disposition", schema = "public"))] + pub struct DbBpDatasetDispositionEnum; + + /// This type is not actually public, because [`BlueprintDatasetDisposition`] + /// interacts with external logic. + /// + /// However, it must be marked `pub` to avoid errors like `crate-private + /// type `BpDatasetDispositionEnum` in public interface`. Marking this type `pub`, + /// without actually making it public, tricks rustc in a desirable way. + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq)] + #[diesel(sql_type = DbBpDatasetDispositionEnum)] + pub enum DbBpDatasetDisposition; + + // Enum values + InService => b"in_service" + Expunged => b"expunged" +); + +/// Converts a [`BlueprintDatasetDisposition`] to a version that can be inserted +/// into a database. +pub fn to_db_bp_dataset_disposition( + disposition: BlueprintDatasetDisposition, +) -> DbBpDatasetDisposition { + match disposition { + BlueprintDatasetDisposition::InService => { + DbBpDatasetDisposition::InService + } + BlueprintDatasetDisposition::Expunged => { + DbBpDatasetDisposition::Expunged + } + } +} + +impl From for BlueprintDatasetDisposition { + fn from(disposition: DbBpDatasetDisposition) -> Self { + match disposition { + DbBpDatasetDisposition::InService => { + BlueprintDatasetDisposition::InService + } + DbBpDatasetDisposition::Expunged => { + BlueprintDatasetDisposition::Expunged + } + } + } +} + +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = bp_sled_omicron_datasets)] +pub struct BpSledOmicronDatasets { + pub blueprint_id: Uuid, + pub sled_id: DbTypedUuid, + pub generation: Generation, +} + +impl BpSledOmicronDatasets { + pub fn new( + blueprint_id: Uuid, + sled_id: SledUuid, + datasets_config: &BlueprintDatasetsConfig, + ) -> Self { + Self { + blueprint_id, + sled_id: sled_id.into(), + generation: Generation(datasets_config.generation), + } + } +} + +/// DB representation of [BlueprintDatasetConfig] +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = bp_omicron_dataset)] +pub struct BpOmicronDataset { + pub blueprint_id: Uuid, + pub sled_id: DbTypedUuid, + pub id: DbTypedUuid, + + pub disposition: DbBpDatasetDisposition, + + pub pool_id: DbTypedUuid, + pub kind: crate::DatasetKind, + zone_name: Option, + pub ip: Option, + pub port: Option, + + pub quota: Option, + pub reservation: Option, + pub compression: String, +} + +impl BpOmicronDataset { + pub fn new( + blueprint_id: Uuid, + sled_id: SledUuid, + dataset_config: &BlueprintDatasetConfig, + ) -> Self { + Self { + blueprint_id, + sled_id: sled_id.into(), + id: dataset_config.id.into(), + disposition: to_db_bp_dataset_disposition( + dataset_config.disposition, + ), + pool_id: dataset_config.pool.id().into(), + kind: (&dataset_config.kind).into(), + zone_name: dataset_config.kind.zone_name().map(String::from), + ip: dataset_config.address.map(|addr| addr.ip().into()), + port: dataset_config.address.map(|addr| addr.port().into()), + quota: dataset_config.quota.map(|q| q.into()), + reservation: dataset_config.reservation.map(|r| r.into()), + compression: dataset_config.compression.to_string(), + } + } +} + +impl TryFrom for BlueprintDatasetConfig { + type Error = anyhow::Error; + + fn try_from(dataset: BpOmicronDataset) -> Result { + let address = match (dataset.ip, dataset.port) { + (Some(ip), Some(port)) => { + Some(SocketAddrV6::new(ip.into(), port.into(), 0, 0)) + } + (None, None) => None, + (_, _) => anyhow::bail!( + "Either both 'ip' and 'port' should be set, or neither" + ), + }; + + Ok(Self { + disposition: dataset.disposition.into(), + id: dataset.id.into(), + pool: omicron_common::zpool_name::ZpoolName::new_external( + dataset.pool_id.into(), + ), + kind: crate::DatasetKind::try_into_api( + dataset.kind, + dataset.zone_name, + )?, + address, + quota: dataset.quota.map(|b| b.into()), + reservation: dataset.reservation.map(|b| b.into()), + compression: dataset.compression.parse()?, + }) + } +} + /// See [`nexus_types::deployment::BlueprintZonesConfig`]. #[derive(Queryable, Clone, Debug, Selectable, Insertable)] #[diesel(table_name = bp_sled_omicron_zones)] diff --git a/nexus/db-model/src/instance.rs b/nexus/db-model/src/instance.rs index e7aa989971..b9d08191aa 100644 --- a/nexus/db-model/src/instance.rs +++ b/nexus/db-model/src/instance.rs @@ -259,7 +259,9 @@ pub struct InstanceAutoRestart { /// /// This indicates whether the instance should be automatically restarted by /// the control plane on failure. If this is `NULL`, no auto-restart policy - /// has been configured for this instance by the user. + /// has been configured for this instance by the user. In that case, the + /// control plane will use the default policy when determining whether + /// this instance can be automatically restarted. #[diesel(column_name = auto_restart_policy)] #[serde(default)] pub policy: Option, diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 84e0c7e4f2..8a97e43639 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -145,6 +145,7 @@ table! { speed -> crate::SwitchLinkSpeedEnum, autoneg -> Bool, lldp_link_config_id -> Nullable, + tx_eq_config_id -> Nullable, } } @@ -164,6 +165,17 @@ table! { } } +table! { + tx_eq_config (id) { + id -> Uuid, + pre1 -> Nullable, + pre2 -> Nullable, + main -> Nullable, + post2 -> Nullable, + post1 -> Nullable, + } +} + table! { switch_port_settings_interface_config (id) { port_settings_id -> Uuid, @@ -1038,6 +1050,10 @@ table! { kind -> crate::DatasetKindEnum, size_used -> Nullable, zone_name -> Nullable, + + quota -> Nullable, + reservation -> Nullable, + compression -> Nullable, } } @@ -1649,6 +1665,35 @@ table! { } } +table! { + bp_sled_omicron_datasets (blueprint_id, sled_id) { + blueprint_id -> Uuid, + sled_id -> Uuid, + + generation -> Int8, + } +} + +table! { + bp_omicron_dataset (blueprint_id, id) { + blueprint_id -> Uuid, + sled_id -> Uuid, + id -> Uuid, + + disposition -> crate::DbBpDatasetDispositionEnum, + + pool_id -> Uuid, + kind -> crate::DatasetKindEnum, + zone_name -> Nullable, + ip -> Nullable, + port -> Nullable, + + quota -> Nullable, + reservation -> Nullable, + compression -> Text, + } +} + table! { bp_sled_omicron_zones (blueprint_id, sled_id) { blueprint_id -> Uuid, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index b09a8b7a6b..14eba1cb1c 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(111, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(113, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,8 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(113, "add-tx-eq"), + KnownVersion::new(112, "blueprint-dataset"), KnownVersion::new(111, "drop-omicron-zone-underlay-address"), KnownVersion::new(110, "clickhouse-policy"), KnownVersion::new(109, "inv-clickhouse-keeper-membership"), diff --git a/nexus/db-model/src/switch_port.rs b/nexus/db-model/src/switch_port.rs index 2420482cce..2db9ef04df 100644 --- a/nexus/db-model/src/switch_port.rs +++ b/nexus/db-model/src/switch_port.rs @@ -11,6 +11,7 @@ use crate::schema::{ switch_port_settings_group, switch_port_settings_groups, switch_port_settings_interface_config, switch_port_settings_link_config, switch_port_settings_port_config, switch_port_settings_route_config, + tx_eq_config, }; use crate::{impl_enum_type, SqlU32}; use crate::{SqlU16, SqlU8}; @@ -387,9 +388,11 @@ pub struct SwitchPortLinkConfig { pub fec: SwitchLinkFec, pub speed: SwitchLinkSpeed, pub autoneg: bool, + pub tx_eq_config_id: Option, } impl SwitchPortLinkConfig { + #[allow(clippy::too_many_arguments)] pub fn new( port_settings_id: Uuid, lldp_link_config_id: Uuid, @@ -398,6 +401,7 @@ impl SwitchPortLinkConfig { fec: SwitchLinkFec, speed: SwitchLinkSpeed, autoneg: bool, + tx_eq_config_id: Option, ) -> Self { Self { port_settings_id, @@ -407,6 +411,7 @@ impl SwitchPortLinkConfig { speed, autoneg, mtu: mtu.into(), + tx_eq_config_id, } } } @@ -416,6 +421,7 @@ impl Into for SwitchPortLinkConfig { external::SwitchPortLinkConfig { port_settings_id: self.port_settings_id, lldp_link_config_id: self.lldp_link_config_id, + tx_eq_config_id: self.tx_eq_config_id, link_name: self.link_name.clone(), mtu: self.mtu.into(), fec: self.fec.into(), @@ -494,6 +500,52 @@ impl Into for LldpLinkConfig { } } +#[derive( + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, +)] +#[diesel(table_name = tx_eq_config)] +pub struct TxEqConfig { + pub id: Uuid, + pub pre1: Option, + pub pre2: Option, + pub main: Option, + pub post2: Option, + pub post1: Option, +} + +impl TxEqConfig { + pub fn new( + pre1: Option, + pre2: Option, + main: Option, + post2: Option, + post1: Option, + ) -> Self { + Self { id: Uuid::new_v4(), pre1, pre2, main, post2, post1 } + } +} + +// This converts the internal database version of the config into the +// user-facing version. +impl Into for TxEqConfig { + fn into(self) -> external::TxEqConfig { + external::TxEqConfig { + pre1: self.pre1, + pre2: self.pre2, + main: self.main, + post2: self.post2, + post1: self.post1, + } + } +} + #[derive( Queryable, Insertable, diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index a8d7983959..db2b70488d 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -69,7 +69,7 @@ omicron-workspace-hack.workspace = true omicron-test-utils = { workspace = true, optional = true } [features] -# Enable to export `datastore_test` +# Enable to export `TestDatabase` testing = ["omicron-test-utils"] [dev-dependencies] diff --git a/nexus/db-queries/src/db/collection_attach.rs b/nexus/db-queries/src/db/collection_attach.rs index 127861b2a8..01ad700177 100644 --- a/nexus/db-queries/src/db/collection_attach.rs +++ b/nexus/db-queries/src/db/collection_attach.rs @@ -577,8 +577,8 @@ where #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::identity::Resource as IdentityResource; + use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use chrono::Utc; use db_macros::Resource; diff --git a/nexus/db-queries/src/db/collection_detach.rs b/nexus/db-queries/src/db/collection_detach.rs index cdf8e111c7..618663c331 100644 --- a/nexus/db-queries/src/db/collection_detach.rs +++ b/nexus/db-queries/src/db/collection_detach.rs @@ -481,8 +481,8 @@ where mod test { use super::*; use crate::db::collection_attach::DatastoreAttachTarget; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::identity::Resource as IdentityResource; + use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use chrono::Utc; use db_macros::Resource; diff --git a/nexus/db-queries/src/db/collection_detach_many.rs b/nexus/db-queries/src/db/collection_detach_many.rs index 15267dd7ee..c105604679 100644 --- a/nexus/db-queries/src/db/collection_detach_many.rs +++ b/nexus/db-queries/src/db/collection_detach_many.rs @@ -479,8 +479,8 @@ where mod test { use super::*; use crate::db::collection_attach::DatastoreAttachTarget; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::identity::Resource as IdentityResource; + use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use chrono::Utc; use db_macros::Resource; diff --git a/nexus/db-queries/src/db/collection_insert.rs b/nexus/db-queries/src/db/collection_insert.rs index 7f8275e594..1be10f4eba 100644 --- a/nexus/db-queries/src/db/collection_insert.rs +++ b/nexus/db-queries/src/db/collection_insert.rs @@ -405,8 +405,8 @@ where #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::identity::Resource as IdentityResource; + use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use chrono::{DateTime, Utc}; use db_macros::Resource; diff --git a/nexus/db-queries/src/db/datastore/allow_list.rs b/nexus/db-queries/src/db/datastore/allow_list.rs index ce839ebcbc..335edf2f23 100644 --- a/nexus/db-queries/src/db/datastore/allow_list.rs +++ b/nexus/db-queries/src/db/datastore/allow_list.rs @@ -83,8 +83,8 @@ impl super::DataStore { #[cfg(test)] mod tests { - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::fixed_data::allow_list::USER_FACING_SERVICES_ALLOW_LIST_ID; + use crate::db::pub_test_utils::TestDatabase; use omicron_common::api::external; use omicron_test_utils::dev; diff --git a/nexus/db-queries/src/db/datastore/bgp.rs b/nexus/db-queries/src/db/datastore/bgp.rs index ccf5c6bb75..7f0607bf72 100644 --- a/nexus/db-queries/src/db/datastore/bgp.rs +++ b/nexus/db-queries/src/db/datastore/bgp.rs @@ -1000,7 +1000,7 @@ impl DataStore { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::Name; use omicron_test_utils::dev; diff --git a/nexus/db-queries/src/db/datastore/clickhouse_policy.rs b/nexus/db-queries/src/db/datastore/clickhouse_policy.rs index cdd0e4127b..e207adab37 100644 --- a/nexus/db-queries/src/db/datastore/clickhouse_policy.rs +++ b/nexus/db-queries/src/db/datastore/clickhouse_policy.rs @@ -175,7 +175,7 @@ impl DataStore { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use nexus_inventory::now_db_precision; use nexus_types::deployment::ClickhouseMode; use omicron_test_utils::dev; diff --git a/nexus/db-queries/src/db/datastore/cockroachdb_node_id.rs b/nexus/db-queries/src/db/datastore/cockroachdb_node_id.rs index 1c1a699c26..4de50a6711 100644 --- a/nexus/db-queries/src/db/datastore/cockroachdb_node_id.rs +++ b/nexus/db-queries/src/db/datastore/cockroachdb_node_id.rs @@ -82,7 +82,7 @@ impl DataStore { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use omicron_test_utils::dev; #[tokio::test] diff --git a/nexus/db-queries/src/db/datastore/cockroachdb_settings.rs b/nexus/db-queries/src/db/datastore/cockroachdb_settings.rs index ba7c302f83..e979b68a19 100644 --- a/nexus/db-queries/src/db/datastore/cockroachdb_settings.rs +++ b/nexus/db-queries/src/db/datastore/cockroachdb_settings.rs @@ -134,7 +134,7 @@ impl DataStore { #[cfg(test)] mod test { use super::CockroachDbSettings; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use nexus_types::deployment::CockroachDbClusterVersion; use omicron_common::api::external::Error; use omicron_test_utils::dev; diff --git a/nexus/db-queries/src/db/datastore/dataset.rs b/nexus/db-queries/src/db/datastore/dataset.rs index 1843df0c7d..996f105254 100644 --- a/nexus/db-queries/src/db/datastore/dataset.rs +++ b/nexus/db-queries/src/db/datastore/dataset.rs @@ -12,6 +12,7 @@ use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; use crate::db::identity::Asset; use crate::db::model::Dataset; @@ -20,18 +21,24 @@ use crate::db::model::PhysicalDiskPolicy; use crate::db::model::Zpool; use crate::db::pagination::paginated; use crate::db::pagination::Paginator; +use crate::db::TransactionError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use diesel::upsert::excluded; +use futures::FutureExt; use nexus_db_model::DatasetKind; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; +use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; +use omicron_uuid_kinds::BlueprintUuid; +use omicron_uuid_kinds::DatasetUuid; +use omicron_uuid_kinds::GenericUuid; use uuid::Uuid; impl DataStore { @@ -53,6 +60,45 @@ impl DataStore { &self, dataset: Dataset, ) -> CreateResult { + let conn = &*self.pool_connection_unauthorized().await?; + Self::dataset_upsert_on_connection(&conn, dataset).await.map_err(|e| { + match e { + TransactionError::CustomError(e) => e, + TransactionError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + } + }) + } + + pub async fn dataset_upsert_if_blueprint_is_current_target( + &self, + opctx: &OpContext, + bp_id: BlueprintUuid, + dataset: Dataset, + ) -> CreateResult { + let conn = self.pool_connection_unauthorized().await?; + + self.transaction_if_current_blueprint_is( + &conn, + "dataset_upsert_if_blueprint_is_current_target", + opctx, + bp_id, + |conn| { + let dataset = dataset.clone(); + async move { + Self::dataset_upsert_on_connection(&conn, dataset).await + } + .boxed() + }, + ) + .await + } + + async fn dataset_upsert_on_connection( + conn: &async_bb8_diesel::Connection, + dataset: Dataset, + ) -> Result> { use db::schema::dataset::dsl; let dataset_id = dataset.id(); @@ -69,24 +115,33 @@ impl DataStore { dsl::ip.eq(excluded(dsl::ip)), dsl::port.eq(excluded(dsl::port)), dsl::kind.eq(excluded(dsl::kind)), + dsl::zone_name.eq(excluded(dsl::zone_name)), + dsl::quota.eq(excluded(dsl::quota)), + dsl::reservation.eq(excluded(dsl::reservation)), + dsl::compression.eq(excluded(dsl::compression)), )), ) - .insert_and_get_result_async( - &*self.pool_connection_unauthorized().await?, - ) + .insert_and_get_result_async(conn) .await .map_err(|e| match e { - AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { - type_name: ResourceType::Zpool, - lookup_type: LookupType::ById(zpool_id), - }, - AsyncInsertError::DatabaseError(e) => public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Dataset, - &dataset_id.to_string(), - ), - ), + AsyncInsertError::CollectionNotFound => { + TransactionError::CustomError(Error::ObjectNotFound { + type_name: ResourceType::Zpool, + lookup_type: LookupType::ById(zpool_id), + }) + } + AsyncInsertError::DatabaseError(e) => { + if retryable(&e) { + return TransactionError::Database(e); + } + TransactionError::CustomError(public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Dataset, + &dataset_id.to_string(), + ), + )) + } }) } @@ -183,6 +238,63 @@ impl DataStore { Ok(all_datasets) } + pub async fn dataset_delete( + &self, + opctx: &OpContext, + id: DatasetUuid, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let conn = self.pool_connection_authorized(&opctx).await?; + + Self::dataset_delete_on_connection(&conn, id) + .await + .map_err(|e| e.into()) + } + + pub async fn dataset_delete_if_blueprint_is_current_target( + &self, + opctx: &OpContext, + bp_id: BlueprintUuid, + id: DatasetUuid, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let conn = self.pool_connection_authorized(&opctx).await?; + + self.transaction_if_current_blueprint_is( + &conn, + "dataset_delete_if_blueprint_is_current_target", + opctx, + bp_id, + |conn| { + async move { + Self::dataset_delete_on_connection(&conn, id).await + } + .boxed() + }, + ) + .await + } + + async fn dataset_delete_on_connection( + conn: &async_bb8_diesel::Connection, + id: DatasetUuid, + ) -> Result<(), TransactionError> { + use db::schema::dataset::dsl as dataset_dsl; + let now = Utc::now(); + + let id = *id.as_untyped_uuid(); + diesel::update(dataset_dsl::dataset) + .filter(dataset_dsl::time_deleted.is_null()) + .filter(dataset_dsl::id.eq(id)) + .set(dataset_dsl::time_deleted.eq(now)) + .execute_async(conn) + .await + .map(|_rows_modified| ()) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } + pub async fn dataset_physical_disk_in_service( &self, dataset_id: Uuid, @@ -235,30 +347,27 @@ impl DataStore { #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use nexus_db_model::Generation; use nexus_db_model::SledBaseboard; use nexus_db_model::SledSystemHardware; use nexus_db_model::SledUpdate; + use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; + use nexus_types::deployment::Blueprint; + use nexus_types::deployment::BlueprintTarget; use omicron_common::api::internal::shared::DatasetKind as ApiDatasetKind; use omicron_test_utils::dev; + use omicron_uuid_kinds::SledUuid; + use omicron_uuid_kinds::ZpoolUuid; - #[tokio::test] - async fn test_insert_if_not_exists() { - let logctx = dev::test_setup_log("inventory_insert"); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - - // There should be no datasets initially. - assert_eq!( - datastore.dataset_list_all_batched(opctx, None).await.unwrap(), - [] - ); - + async fn create_sled_and_zpool( + datastore: &DataStore, + opctx: &OpContext, + ) -> (SledUuid, ZpoolUuid) { // Create a fake sled that holds our fake zpool. - let sled_id = Uuid::new_v4(); + let sled_id = SledUuid::new_v4(); let sled = SledUpdate::new( - sled_id, + *sled_id.as_untyped_uuid(), "[::1]:0".parse().unwrap(), SledBaseboard { serial_number: "test-sn".to_string(), @@ -277,18 +386,40 @@ mod test { datastore.sled_upsert(sled).await.expect("failed to upsert sled"); // Create a fake zpool that backs our fake datasets. - let zpool_id = Uuid::new_v4(); - let zpool = Zpool::new(zpool_id, sled_id, Uuid::new_v4()); + let zpool_id = ZpoolUuid::new_v4(); + let zpool = Zpool::new( + *zpool_id.as_untyped_uuid(), + *sled_id.as_untyped_uuid(), + Uuid::new_v4(), + ); datastore .zpool_insert(opctx, zpool) .await .expect("failed to upsert zpool"); + (sled_id, zpool_id) + } + + #[tokio::test] + async fn test_insert_if_not_exists() { + let logctx = dev::test_setup_log("insert_if_not_exists"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + // There should be no datasets initially. + assert_eq!( + datastore.dataset_list_all_batched(opctx, None).await.unwrap(), + [] + ); + + let (_sled_id, zpool_id) = + create_sled_and_zpool(&datastore, opctx).await; + // Inserting a new dataset should succeed. let dataset1 = datastore .dataset_insert_if_not_exists(Dataset::new( Uuid::new_v4(), - zpool_id, + *zpool_id.as_untyped_uuid(), Some("[::1]:0".parse().unwrap()), ApiDatasetKind::Crucible, )) @@ -321,7 +452,7 @@ mod test { let insert_again_result = datastore .dataset_insert_if_not_exists(Dataset::new( dataset1.id(), - zpool_id, + *zpool_id.as_untyped_uuid(), Some("[::1]:12345".parse().unwrap()), ApiDatasetKind::Cockroach, )) @@ -337,7 +468,7 @@ mod test { let dataset2 = datastore .dataset_upsert(Dataset::new( Uuid::new_v4(), - zpool_id, + *zpool_id.as_untyped_uuid(), Some("[::1]:0".parse().unwrap()), ApiDatasetKind::Cockroach, )) @@ -369,7 +500,7 @@ mod test { let insert_again_result = datastore .dataset_insert_if_not_exists(Dataset::new( dataset1.id(), - zpool_id, + *zpool_id.as_untyped_uuid(), Some("[::1]:12345".parse().unwrap()), ApiDatasetKind::Cockroach, )) @@ -384,4 +515,114 @@ mod test { db.terminate().await; logctx.cleanup_successful(); } + + async fn bp_insert_and_make_target( + opctx: &OpContext, + datastore: &DataStore, + bp: &Blueprint, + ) { + datastore + .blueprint_insert(opctx, bp) + .await + .expect("inserted blueprint"); + datastore + .blueprint_target_set_current( + opctx, + BlueprintTarget { + target_id: bp.id, + enabled: true, + time_made_target: Utc::now(), + }, + ) + .await + .expect("made blueprint the target"); + } + + fn new_dataset_on(zpool_id: ZpoolUuid) -> Dataset { + Dataset::new( + Uuid::new_v4(), + *zpool_id.as_untyped_uuid(), + Some("[::1]:0".parse().unwrap()), + ApiDatasetKind::Cockroach, + ) + } + + #[tokio::test] + async fn test_upsert_and_delete_while_blueprint_changes() { + let logctx = + dev::test_setup_log("upsert_and_delete_while_blueprint_changes"); + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); + + let (sled_id, zpool_id) = + create_sled_and_zpool(&datastore, opctx).await; + + // The datastore methods don't actually read the blueprint, but they do + // guard against concurrent changes to the current target. + // + // We can test behavior by swapping between empty blueprints. + let bp0 = BlueprintBuilder::build_empty_with_sleds( + [sled_id].into_iter(), + "test", + ); + bp_insert_and_make_target(&opctx, &datastore, &bp0).await; + + let bp1 = { + let mut bp1 = bp0.clone(); + bp1.id = Uuid::new_v4(); + bp1.parent_blueprint_id = Some(bp0.id); + bp1 + }; + bp_insert_and_make_target(&opctx, &datastore, &bp1).await; + + let old_blueprint_id = BlueprintUuid::from_untyped_uuid(bp0.id); + let current_blueprint_id = BlueprintUuid::from_untyped_uuid(bp1.id); + + // Upsert referencing old blueprint: Error + datastore + .dataset_upsert_if_blueprint_is_current_target( + &opctx, + old_blueprint_id, + new_dataset_on(zpool_id), + ) + .await + .expect_err( + "Shouldn't be able to insert referencing old blueprint", + ); + + // Upsert referencing current blueprint: OK + let dataset = datastore + .dataset_upsert_if_blueprint_is_current_target( + &opctx, + current_blueprint_id, + new_dataset_on(zpool_id), + ) + .await + .expect("Should be able to insert while blueprint is active"); + + // Delete referencing old blueprint: Error + datastore + .dataset_delete_if_blueprint_is_current_target( + &opctx, + old_blueprint_id, + DatasetUuid::from_untyped_uuid(dataset.id()), + ) + .await + .expect_err( + "Shouldn't be able to delete referencing old blueprint", + ); + + // Delete referencing current blueprint: OK + datastore + .dataset_delete_if_blueprint_is_current_target( + &opctx, + current_blueprint_id, + DatasetUuid::from_untyped_uuid(dataset.id()), + ) + .await + .expect("Should be able to delete while blueprint is active"); + + db.terminate().await; + logctx.cleanup_successful(); + } } diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs index fbb6cd35c8..985a0453c6 100644 --- a/nexus/db-queries/src/db/datastore/db_metadata.rs +++ b/nexus/db-queries/src/db/datastore/db_metadata.rs @@ -496,7 +496,7 @@ impl DataStore { #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use camino::Utf8Path; use camino_tempfile::Utf8TempDir; use nexus_db_model::SCHEMA_VERSION; diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 34fcd45f96..f168c2f8e3 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -21,6 +21,8 @@ use async_bb8_diesel::AsyncRunQueryDsl; use chrono::DateTime; use chrono::Utc; use clickhouse_admin_types::{KeeperId, ServerId}; +use core::future::Future; +use core::pin::Pin; use diesel::expression::SelectableHelper; use diesel::pg::Pg; use diesel::query_builder::AstPass; @@ -36,18 +38,22 @@ use diesel::IntoSql; use diesel::OptionalExtension; use diesel::QueryDsl; use diesel::RunQueryDsl; +use futures::FutureExt; use nexus_db_model::Blueprint as DbBlueprint; use nexus_db_model::BpClickhouseClusterConfig; use nexus_db_model::BpClickhouseKeeperZoneIdToNodeId; use nexus_db_model::BpClickhouseServerZoneIdToNodeId; +use nexus_db_model::BpOmicronDataset; use nexus_db_model::BpOmicronPhysicalDisk; use nexus_db_model::BpOmicronZone; use nexus_db_model::BpOmicronZoneNic; +use nexus_db_model::BpSledOmicronDatasets; use nexus_db_model::BpSledOmicronPhysicalDisks; use nexus_db_model::BpSledOmicronZones; use nexus_db_model::BpSledState; use nexus_db_model::BpTarget; use nexus_types::deployment::Blueprint; +use nexus_types::deployment::BlueprintDatasetsConfig; use nexus_types::deployment::BlueprintMetadata; use nexus_types::deployment::BlueprintPhysicalDisksConfig; use nexus_types::deployment::BlueprintTarget; @@ -62,6 +68,7 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_common::bail_unless; +use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; @@ -102,6 +109,76 @@ impl DataStore { Self::blueprint_insert_on_connection(&conn, opctx, blueprint).await } + /// Creates a transaction iff the current blueprint is "bp_id". + /// + /// - The transaction is retryable and named "name" + /// - The "bp_id" value is checked as the first operation within the + /// transaction. + /// - If "bp_id" is still the current target, then "f" is called, + /// within a transactional context. + pub async fn transaction_if_current_blueprint_is( + &self, + conn: &async_bb8_diesel::Connection, + name: &'static str, + opctx: &OpContext, + bp_id: BlueprintUuid, + f: Func, + ) -> Result + where + Func: for<'t> Fn( + &'t async_bb8_diesel::Connection, + ) -> Pin< + Box< + dyn Future>> + + Send + + 't, + >, + > + Send + + Sync + + Clone, + R: Send + 'static, + { + let err = OptionalError::new(); + let r = self + .transaction_retry_wrapper(name) + .transaction(&conn, |conn| { + let err = err.clone(); + let f = f.clone(); + async move { + // Bail if `bp_id` is no longer the target + let target = + Self::blueprint_target_get_current_on_connection( + &conn, opctx, + ) + .await + .map_err(|txn_error| txn_error.into_diesel(&err))?; + let bp_id_current = + BlueprintUuid::from_untyped_uuid(target.target_id); + if bp_id_current != bp_id { + return Err(err.bail( + Error::invalid_request(format!( + "blueprint target has changed from {} -> {}", + bp_id, bp_id_current + )) + .into(), + )); + } + + // Otherwise, perform our actual operation + f(&conn) + .await + .map_err(|txn_error| txn_error.into_diesel(&err)) + } + .boxed() + }) + .await + .map_err(|e| match err.take() { + Some(txn_error) => txn_error.into(), + None => public_error_from_diesel(e, ErrorHandler::Server), + })?; + Ok(r) + } + /// Variant of [Self::blueprint_insert] which may be called from a /// transaction context. pub(crate) async fn blueprint_insert_on_connection( @@ -156,6 +233,28 @@ impl DataStore { }) }) .collect::>(); + + let sled_omicron_datasets = blueprint + .blueprint_datasets + .iter() + .map(|(sled_id, datasets_config)| { + BpSledOmicronDatasets::new( + blueprint_id, + *sled_id, + datasets_config, + ) + }) + .collect::>(); + let omicron_datasets = blueprint + .blueprint_datasets + .iter() + .flat_map(|(sled_id, datasets_config)| { + datasets_config.datasets.values().map(move |dataset| { + BpOmicronDataset::new(blueprint_id, *sled_id, dataset) + }) + }) + .collect::>(); + let sled_omicron_zones = blueprint .blueprint_zones .iter() @@ -274,6 +373,24 @@ impl DataStore { .await?; } + // Insert all datasets for this blueprint. + + { + use db::schema::bp_sled_omicron_datasets::dsl as sled_datasets; + let _ = diesel::insert_into(sled_datasets::bp_sled_omicron_datasets) + .values(sled_omicron_datasets) + .execute_async(&conn) + .await?; + } + + { + use db::schema::bp_omicron_dataset::dsl as omicron_dataset; + let _ = diesel::insert_into(omicron_dataset::bp_omicron_dataset) + .values(omicron_datasets) + .execute_async(&conn) + .await?; + } + // Insert all the Omicron zones for this blueprint. { use db::schema::bp_sled_omicron_zones::dsl as sled_zones; @@ -522,6 +639,50 @@ impl DataStore { blueprint_physical_disks }; + // Do the same thing we just did for zones, but for datasets too. + let mut blueprint_datasets: BTreeMap< + SledUuid, + BlueprintDatasetsConfig, + > = { + use db::schema::bp_sled_omicron_datasets::dsl; + + let mut blueprint_datasets = BTreeMap::new(); + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + while let Some(p) = paginator.next() { + let batch = paginated( + dsl::bp_sled_omicron_datasets, + dsl::sled_id, + &p.current_pagparams(), + ) + .filter(dsl::blueprint_id.eq(blueprint_id)) + .select(BpSledOmicronDatasets::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + paginator = p.found_batch(&batch, &|s| s.sled_id); + + for s in batch { + let old = blueprint_datasets.insert( + s.sled_id.into(), + BlueprintDatasetsConfig { + generation: *s.generation, + datasets: BTreeMap::new(), + }, + ); + bail_unless!( + old.is_none(), + "found duplicate sled ID in bp_sled_omicron_datasets: {}", + s.sled_id + ); + } + } + + blueprint_datasets + }; + // Assemble a mutable map of all the NICs found, by NIC id. As we // match these up with the corresponding zone below, we'll remove items // from this set. That way we can tell if the same NIC was used twice @@ -685,6 +846,58 @@ impl DataStore { } } + // Load all the datasets for each sled + { + use db::schema::bp_omicron_dataset::dsl; + + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + while let Some(p) = paginator.next() { + // `paginated` implicitly orders by our `id`, which is also + // handy for testing: the datasets are always consistently ordered + let batch = paginated( + dsl::bp_omicron_dataset, + dsl::id, + &p.current_pagparams(), + ) + .filter(dsl::blueprint_id.eq(blueprint_id)) + .select(BpOmicronDataset::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + paginator = p.found_batch(&batch, &|d| d.id); + + for d in batch { + let sled_datasets = blueprint_datasets + .get_mut(&d.sled_id.into()) + .ok_or_else(|| { + // This error means that we found a row in + // bp_omicron_dataset with no associated record in + // bp_sled_omicron_datasets. This should be + // impossible and reflects either a bug or database + // corruption. + Error::internal_error(&format!( + "dataset {}: unknown sled: {}", + d.id, d.sled_id + )) + })?; + + let dataset_id = d.id; + sled_datasets.datasets.insert( + dataset_id.into(), + d.try_into().map_err(|e| { + Error::internal_error(&format!( + "Cannot parse dataset {}: {e}", + dataset_id + )) + })?, + ); + } + } + } + // Sort all disks to match what blueprint builders do. for (_, disks_config) in blueprint_disks.iter_mut() { disks_config.disks.sort_unstable_by_key(|d| d.id); @@ -844,6 +1057,7 @@ impl DataStore { id: blueprint_id, blueprint_zones, blueprint_disks, + blueprint_datasets, sled_state, parent_blueprint_id, internal_dns_version, @@ -879,6 +1093,8 @@ impl DataStore { nsled_states, nsled_physical_disks, nphysical_disks, + nsled_datasets, + ndatasets, nsled_agent_zones, nzones, nnics, @@ -890,7 +1106,7 @@ impl DataStore { // Ensure that blueprint we're about to delete is not the // current target. let current_target = - self.blueprint_current_target_only(&conn).await?; + Self::blueprint_current_target_only(&conn).await?; if current_target.target_id == blueprint_id { return Err(TransactionError::CustomError( Error::conflict(format!( @@ -950,6 +1166,26 @@ impl DataStore { .await? }; + // Remove rows associated with Omicron datasets + let nsled_datasets = { + use db::schema::bp_sled_omicron_datasets::dsl; + diesel::delete( + dsl::bp_sled_omicron_datasets + .filter(dsl::blueprint_id.eq(blueprint_id)), + ) + .execute_async(&conn) + .await? + }; + let ndatasets = { + use db::schema::bp_omicron_dataset::dsl; + diesel::delete( + dsl::bp_omicron_dataset + .filter(dsl::blueprint_id.eq(blueprint_id)), + ) + .execute_async(&conn) + .await? + }; + // Remove rows associated with Omicron zones let nsled_agent_zones = { use db::schema::bp_sled_omicron_zones::dsl; @@ -1014,6 +1250,8 @@ impl DataStore { nsled_states, nsled_physical_disks, nphysical_disks, + nsled_datasets, + ndatasets, nsled_agent_zones, nzones, nnics, @@ -1036,6 +1274,8 @@ impl DataStore { "nsled_states" => nsled_states, "nsled_physical_disks" => nsled_physical_disks, "nphysical_disks" => nphysical_disks, + "nsled_datasets" => nsled_datasets, + "ndatasets" => ndatasets, "nsled_agent_zones" => nsled_agent_zones, "nzones" => nzones, "nnics" => nnics, @@ -1102,15 +1342,17 @@ impl DataStore { async move { // Bail out if `blueprint` isn't the current target. - let current_target = self - .blueprint_current_target_only(&conn) + let current_target = Self::blueprint_current_target_only(&conn) .await .map_err(|e| err.bail(e))?; if current_target.target_id != blueprint.id { - return Err(err.bail(Error::invalid_request(format!( + return Err(err.bail( + Error::invalid_request(format!( "blueprint {} is not the current target blueprint ({})", blueprint.id, current_target.target_id - )))); + )) + .into(), + )); } // See the comment on this method; this lets us notify our test @@ -1151,7 +1393,7 @@ impl DataStore { .map(|(_sled_id, zone)| zone), ) .await - .map_err(|e| err.bail(e))?; + .map_err(|e| err.bail(e.into()))?; self.ensure_zone_external_networking_allocated_on_connection( &conn, opctx, @@ -1162,7 +1404,7 @@ impl DataStore { .map(|(_sled_id, zone)| zone), ) .await - .map_err(|e| err.bail(e))?; + .map_err(|e| err.bail(e.into()))?; Ok(()) } @@ -1170,7 +1412,7 @@ impl DataStore { .await .map_err(|e| { if let Some(err) = err.take() { - err + err.into() } else { public_error_from_diesel(e, ErrorHandler::Server) } @@ -1320,7 +1562,7 @@ impl DataStore { opctx.authorize(authz::Action::Read, &authz::BLUEPRINT_CONFIG).await?; let conn = self.pool_connection_authorized(opctx).await?; - let target = self.blueprint_current_target_only(&conn).await?; + let target = Self::blueprint_current_target_only(&conn).await?; // The blueprint for the current target cannot be deleted while it is // the current target, but it's possible someone else (a) made a new @@ -1334,6 +1576,15 @@ impl DataStore { Ok((target, blueprint)) } + /// Get the current target blueprint, if one exists + pub async fn blueprint_target_get_current_on_connection( + conn: &async_bb8_diesel::Connection, + opctx: &OpContext, + ) -> Result> { + opctx.authorize(authz::Action::Read, &authz::BLUEPRINT_CONFIG).await?; + Self::blueprint_current_target_only(&conn).await + } + /// Get the current target blueprint, if one exists pub async fn blueprint_target_get_current( &self, @@ -1341,7 +1592,7 @@ impl DataStore { ) -> Result { opctx.authorize(authz::Action::Read, &authz::BLUEPRINT_CONFIG).await?; let conn = self.pool_connection_authorized(opctx).await?; - self.blueprint_current_target_only(&conn).await + Self::blueprint_current_target_only(&conn).await.map_err(|e| e.into()) } // Helper to fetch the current blueprint target (without fetching the entire @@ -1349,9 +1600,8 @@ impl DataStore { // // Caller is responsible for checking authz for this operation. async fn blueprint_current_target_only( - &self, conn: &async_bb8_diesel::Connection, - ) -> Result { + ) -> Result> { use db::schema::bp_target::dsl; let current_target = dsl::bp_target @@ -1730,7 +1980,7 @@ impl RunQueryDsl for InsertTargetQuery {} mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use crate::db::raw_query_builder::QueryBuilder; use nexus_inventory::now_db_precision; use nexus_inventory::CollectionBuilder; @@ -1821,6 +2071,12 @@ mod tests { for (table_name, result) in [ query_count!(blueprint, id), + query_count!(bp_sled_state, blueprint_id), + query_count!(bp_sled_omicron_datasets, blueprint_id), + query_count!(bp_sled_omicron_physical_disks, blueprint_id), + query_count!(bp_sled_omicron_zones, blueprint_id), + query_count!(bp_omicron_dataset, blueprint_id), + query_count!(bp_omicron_physical_disk, blueprint_id), query_count!(bp_omicron_zone, blueprint_id), query_count!(bp_omicron_zone_nic, blueprint_id), ] { @@ -1840,16 +2096,20 @@ mod tests { .map(|i| { ( ZpoolUuid::new_v4(), - SledDisk { - disk_identity: DiskIdentity { - vendor: String::from("v"), - serial: format!("s-{i}"), - model: String::from("m"), + ( + SledDisk { + disk_identity: DiskIdentity { + vendor: String::from("v"), + serial: format!("s-{i}"), + model: String::from("m"), + }, + disk_id: PhysicalDiskUuid::new_v4(), + policy: PhysicalDiskPolicy::InService, + state: PhysicalDiskState::Active, }, - disk_id: PhysicalDiskUuid::new_v4(), - policy: PhysicalDiskPolicy::InService, - state: PhysicalDiskState::Active, - }, + // Datasets + vec![], + ), ) }) .collect(); @@ -2074,7 +2334,12 @@ mod tests { .resources, ) .unwrap(), - EnsureMultiple::Changed { added: 4, removed: 0 } + EnsureMultiple::Changed { + added: 4, + updated: 0, + expunged: 0, + removed: 0 + } ); // Add zones to our new sled. diff --git a/nexus/db-queries/src/db/datastore/deployment/external_networking.rs b/nexus/db-queries/src/db/datastore/deployment/external_networking.rs index d2c8d5c27f..99d7cffdfb 100644 --- a/nexus/db-queries/src/db/datastore/deployment/external_networking.rs +++ b/nexus/db-queries/src/db/datastore/deployment/external_networking.rs @@ -408,7 +408,7 @@ impl DataStore { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use crate::db::queries::ALLOW_FULL_TABLE_SCAN_SQL; use anyhow::Context as _; use async_bb8_diesel::AsyncSimpleConnection; @@ -869,8 +869,7 @@ mod tests { #[tokio::test] async fn test_allocate_external_networking() { // Set up. - usdt::register_probes().unwrap(); - let logctx = dev::test_setup_log("test_service_ip_list"); + let logctx = dev::test_setup_log("test_allocate_external_networking"); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); @@ -1130,8 +1129,7 @@ mod tests { #[tokio::test] async fn test_deallocate_external_networking() { // Set up. - usdt::register_probes().unwrap(); - let logctx = dev::test_setup_log("test_service_ip_list"); + let logctx = dev::test_setup_log("test_deallocate_external_networking"); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); diff --git a/nexus/db-queries/src/db/datastore/disk.rs b/nexus/db-queries/src/db/datastore/disk.rs index 76f4055373..572224161d 100644 --- a/nexus/db-queries/src/db/datastore/disk.rs +++ b/nexus/db-queries/src/db/datastore/disk.rs @@ -843,7 +843,7 @@ impl DataStore { mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use nexus_types::external_api::params; use omicron_common::api::external; use omicron_test_utils::dev; diff --git a/nexus/db-queries/src/db/datastore/dns.rs b/nexus/db-queries/src/db/datastore/dns.rs index 9279933e47..a691ce43aa 100644 --- a/nexus/db-queries/src/db/datastore/dns.rs +++ b/nexus/db-queries/src/db/datastore/dns.rs @@ -729,8 +729,8 @@ impl DataStoreDnsTest for DataStore { #[cfg(test)] mod test { - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::DnsVersionUpdateBuilder; + use crate::db::pub_test_utils::TestDatabase; use crate::db::DataStore; use crate::db::TransactionError; use assert_matches::assert_matches; diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs index a03b6a6249..9447d552e4 100644 --- a/nexus/db-queries/src/db/datastore/external_ip.rs +++ b/nexus/db-queries/src/db/datastore/external_ip.rs @@ -1132,7 +1132,7 @@ impl DataStore { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use nexus_types::deployment::OmicronZoneExternalFloatingIp; use nexus_types::deployment::OmicronZoneExternalSnatIp; use nexus_types::external_api::shared::IpRange; @@ -1161,7 +1161,6 @@ mod tests { #[tokio::test] async fn test_service_ip_list() { - usdt::register_probes().unwrap(); let logctx = dev::test_setup_log("test_service_ip_list"); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index 0698883891..4ff97e0e9a 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -1943,10 +1943,10 @@ impl DataStore { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::sled; use crate::db::lookup::LookupPath; use crate::db::pagination::Paginator; + use crate::db::pub_test_utils::TestDatabase; use nexus_db_model::InstanceState; use nexus_db_model::Project; use nexus_db_model::VmmRuntimeState; diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 4a2ab216a2..97d4b6357c 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -2444,8 +2444,8 @@ impl DataStoreInventoryTest for DataStore { #[cfg(test)] mod test { use crate::db::datastore::inventory::DataStoreInventoryTest; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::DataStoreConnection; + use crate::db::pub_test_utils::TestDatabase; use crate::db::raw_query_builder::{QueryBuilder, TrustedStr}; use crate::db::schema; use crate::db::DataStore; diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs index 9ea8f7b088..9548003ee5 100644 --- a/nexus/db-queries/src/db/datastore/ip_pool.rs +++ b/nexus/db-queries/src/db/datastore/ip_pool.rs @@ -1121,10 +1121,10 @@ mod test { use std::num::NonZeroU32; use crate::authz; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::model::{ IpPool, IpPoolResource, IpPoolResourceType, Project, }; + use crate::db::pub_test_utils::TestDatabase; use assert_matches::assert_matches; use nexus_types::external_api::params; use nexus_types::identity::Resource; diff --git a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs index 80794f193a..2415c2d654 100644 --- a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs +++ b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs @@ -379,7 +379,7 @@ fn ipv4_nat_next_version() -> diesel::expression::SqlLiteral { mod test { use std::{net::Ipv4Addr, str::FromStr}; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use chrono::Utc; use nexus_db_model::{Ipv4NatEntry, Ipv4NatValues, MacAddr, Vni}; use omicron_common::api::external; diff --git a/nexus/db-queries/src/db/datastore/migration.rs b/nexus/db-queries/src/db/datastore/migration.rs index e1d8c070e7..320146ae78 100644 --- a/nexus/db-queries/src/db/datastore/migration.rs +++ b/nexus/db-queries/src/db/datastore/migration.rs @@ -178,9 +178,9 @@ impl DataStore { mod tests { use super::*; use crate::authz; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::lookup::LookupPath; use crate::db::model::Instance; + use crate::db::pub_test_utils::TestDatabase; use nexus_db_model::Project; use nexus_types::external_api::params; use nexus_types::silo::DEFAULT_SILO_ID; diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index f943e70c4a..5344ffc1d9 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -78,8 +78,6 @@ mod oximeter; mod physical_disk; mod probe; mod project; -#[cfg(any(test, feature = "testing"))] -pub mod pub_test_utils; mod quota; mod rack; mod region; @@ -446,7 +444,6 @@ mod test { use crate::authn; use crate::authn::SiloAuthnPolicy; use crate::authz; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::test_utils::{ IneligibleSledKind, IneligibleSleds, }; @@ -459,6 +456,7 @@ mod test { Region, SiloUser, SledBaseboard, SledSystemHardware, SledUpdate, SshKey, Zpool, }; + use crate::db::pub_test_utils::TestDatabase; use crate::db::queries::vpc_subnet::InsertVpcSubnetQuery; use chrono::{Duration, Utc}; use futures::stream; diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index b7f0622609..e4cf7079d7 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -892,7 +892,7 @@ impl DataStore { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use nexus_db_fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; @@ -920,7 +920,6 @@ mod tests { #[tokio::test] async fn test_service_network_interfaces_list() { - usdt::register_probes().unwrap(); let logctx = dev::test_setup_log("test_service_network_interfaces_list"); let db = TestDatabase::new_with_datastore(&logctx.log).await; diff --git a/nexus/db-queries/src/db/datastore/oximeter.rs b/nexus/db-queries/src/db/datastore/oximeter.rs index be5ddb91bb..2654e8f2d5 100644 --- a/nexus/db-queries/src/db/datastore/oximeter.rs +++ b/nexus/db-queries/src/db/datastore/oximeter.rs @@ -292,7 +292,7 @@ impl DataStore { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use nexus_types::internal_api::params; use omicron_common::api::internal::nexus; use omicron_test_utils::dev; diff --git a/nexus/db-queries/src/db/datastore/physical_disk.rs b/nexus/db-queries/src/db/datastore/physical_disk.rs index 73aa837af8..29c210e5d6 100644 --- a/nexus/db-queries/src/db/datastore/physical_disk.rs +++ b/nexus/db-queries/src/db/datastore/physical_disk.rs @@ -321,12 +321,12 @@ impl DataStore { #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::test::{ sled_baseboard_for_test, sled_system_hardware_for_test, }; use crate::db::lookup::LookupPath; use crate::db::model::{PhysicalDiskKind, Sled, SledUpdate}; + use crate::db::pub_test_utils::TestDatabase; use dropshot::PaginationOrder; use nexus_db_model::Generation; use nexus_sled_agent_shared::inventory::{ diff --git a/nexus/db-queries/src/db/datastore/pub_test_utils.rs b/nexus/db-queries/src/db/datastore/pub_test_utils.rs deleted file mode 100644 index 1e3343a165..0000000000 --- a/nexus/db-queries/src/db/datastore/pub_test_utils.rs +++ /dev/null @@ -1,170 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Test support code that can be enabled by dependencies via this crate's -//! `testing` feature. -//! -//! This feature should only be enabled under `dev-dependencies` to avoid this -//! test support code leaking into release binaries. - -use crate::authz; -use crate::context::OpContext; -use crate::db; -use crate::db::DataStore; -use omicron_test_utils::dev::db::CockroachInstance; -use slog::Logger; -use std::sync::Arc; -use uuid::Uuid; - -#[cfg(test)] -mod test { - use super::*; - use nexus_test_utils::db::test_setup_database; - - enum TestKind { - Pool { pool: Arc }, - RawDatastore { datastore: Arc }, - Datastore { opctx: OpContext, datastore: Arc }, - } - - /// A test database with a pool connected to it. - pub struct TestDatabase { - db: CockroachInstance, - - kind: TestKind, - } - - impl TestDatabase { - /// Creates a new database for test usage, with a pool. - /// - /// [`Self::terminate`] should be called before the test finishes, - /// or dropping the [`TestDatabase`] will panic. - pub async fn new_with_pool(log: &Logger) -> Self { - let db = test_setup_database(log).await; - let cfg = db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(db::Pool::new_single_host(log, &cfg)); - Self { db, kind: TestKind::Pool { pool } } - } - - /// Creates a new database for test usage, with a pre-loaded datastore. - /// - /// [`Self::terminate`] should be called before the test finishes, - /// or dropping the [`TestDatabase`] will panic. - pub async fn new_with_datastore(log: &Logger) -> Self { - let db = test_setup_database(log).await; - let (opctx, datastore) = - crate::db::datastore::test_utils::datastore_test(log, &db) - .await; - - Self { db, kind: TestKind::Datastore { opctx, datastore } } - } - - /// Creates a new database for test usage, with a raw datastore. - /// - /// [`Self::terminate`] should be called before the test finishes, - /// or dropping the [`TestDatabase`] will panic. - pub async fn new_with_raw_datastore(log: &Logger) -> Self { - let db = test_setup_database(log).await; - let cfg = db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(db::Pool::new_single_host(log, &cfg)); - let datastore = - Arc::new(DataStore::new(&log, pool, None).await.unwrap()); - Self { db, kind: TestKind::RawDatastore { datastore } } - } - - pub fn pool(&self) -> &Arc { - match &self.kind { - TestKind::Pool { pool } => pool, - TestKind::RawDatastore { .. } | TestKind::Datastore { .. } => { - panic!("Wrong test type; try using `TestDatabase::new_with_pool`"); - } - } - } - - pub fn opctx(&self) -> &OpContext { - match &self.kind { - TestKind::Pool { .. } | TestKind::RawDatastore { .. } => { - panic!("Wrong test type; try using `TestDatabase::new_with_datastore`"); - } - TestKind::Datastore { opctx, .. } => opctx, - } - } - - pub fn datastore(&self) -> &Arc { - match &self.kind { - TestKind::Pool { .. } => { - panic!("Wrong test type; try using `TestDatabase::new_with_datastore`"); - } - TestKind::RawDatastore { datastore } => datastore, - TestKind::Datastore { datastore, .. } => datastore, - } - } - - /// Shuts down both the database and the pool - pub async fn terminate(mut self) { - match self.kind { - TestKind::Pool { pool } => pool.terminate().await, - TestKind::RawDatastore { datastore } => { - datastore.terminate().await - } - TestKind::Datastore { datastore, .. } => { - datastore.terminate().await - } - } - self.db.cleanup().await.unwrap(); - } - } -} - -#[cfg(test)] -pub use test::TestDatabase; - -/// Constructs a DataStore for use in test suites that has preloaded the -/// built-in users, roles, and role assignments that are needed for basic -/// operation -#[cfg(any(test, feature = "testing"))] -pub async fn datastore_test( - log: &Logger, - db: &CockroachInstance, - rack_id: Uuid, -) -> (OpContext, Arc) { - use crate::authn; - - let cfg = db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(db::Pool::new_single_host(&log, &cfg)); - let datastore = Arc::new(DataStore::new(&log, pool, None).await.unwrap()); - - // Create an OpContext with the credentials of "db-init" just for the - // purpose of loading the built-in users, roles, and assignments. - let opctx = OpContext::for_background( - log.new(o!()), - Arc::new(authz::Authz::new(&log)), - authn::Context::internal_db_init(), - Arc::clone(&datastore) as Arc, - ); - - // TODO: Can we just call "Populate" instead of doing this? - datastore.load_builtin_users(&opctx).await.unwrap(); - datastore.load_builtin_roles(&opctx).await.unwrap(); - datastore.load_builtin_role_asgns(&opctx).await.unwrap(); - datastore.load_builtin_silos(&opctx).await.unwrap(); - datastore.load_builtin_projects(&opctx).await.unwrap(); - datastore.load_builtin_vpcs(&opctx).await.unwrap(); - datastore.load_silo_users(&opctx).await.unwrap(); - datastore.load_silo_user_role_assignments(&opctx).await.unwrap(); - datastore - .load_builtin_fleet_virtual_provisioning_collection(&opctx) - .await - .unwrap(); - datastore.load_builtin_rack_data(&opctx, rack_id).await.unwrap(); - - // Create an OpContext with the credentials of "test-privileged" for general - // testing. - let opctx = OpContext::for_tests( - log.new(o!()), - Arc::clone(&datastore) as Arc, - ); - - (opctx, datastore) -} diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index fe7ee5a6d3..74b3440a7d 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -997,7 +997,6 @@ impl DataStore { #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::test::{ sled_baseboard_for_test, sled_system_hardware_for_test, }; @@ -1006,6 +1005,7 @@ mod test { use crate::db::model::IpKind; use crate::db::model::IpPoolRange; use crate::db::model::Sled; + use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::AsyncSimpleConnection; use internal_dns_types::names::DNS_ZONE; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; @@ -1059,6 +1059,7 @@ mod test { id: Uuid::new_v4(), blueprint_zones: BTreeMap::new(), blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, @@ -1539,6 +1540,7 @@ mod test { sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, parent_blueprint_id: None, @@ -1799,6 +1801,7 @@ mod test { sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, parent_blueprint_id: None, @@ -2010,6 +2013,7 @@ mod test { sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, parent_blueprint_id: None, @@ -2152,6 +2156,7 @@ mod test { sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, parent_blueprint_id: None, diff --git a/nexus/db-queries/src/db/datastore/region_replacement.rs b/nexus/db-queries/src/db/datastore/region_replacement.rs index 508e80a63b..de047d6d0c 100644 --- a/nexus/db-queries/src/db/datastore/region_replacement.rs +++ b/nexus/db-queries/src/db/datastore/region_replacement.rs @@ -895,7 +895,7 @@ impl DataStore { mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use omicron_test_utils::dev; #[tokio::test] diff --git a/nexus/db-queries/src/db/datastore/region_snapshot_replacement.rs b/nexus/db-queries/src/db/datastore/region_snapshot_replacement.rs index d9c8a8b258..6498ef3855 100644 --- a/nexus/db-queries/src/db/datastore/region_snapshot_replacement.rs +++ b/nexus/db-queries/src/db/datastore/region_snapshot_replacement.rs @@ -1059,8 +1059,8 @@ impl DataStore { mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::model::RegionReplacement; + use crate::db::pub_test_utils::TestDatabase; use omicron_test_utils::dev; #[tokio::test] diff --git a/nexus/db-queries/src/db/datastore/saga.rs b/nexus/db-queries/src/db/datastore/saga.rs index f1f0bd18cc..4bc212e997 100644 --- a/nexus/db-queries/src/db/datastore/saga.rs +++ b/nexus/db-queries/src/db/datastore/saga.rs @@ -259,7 +259,7 @@ impl DataStore { #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncSimpleConnection; use db::queries::ALLOW_FULL_TABLE_SCAN_SQL; diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 8e37d7ae7f..89492390d4 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -824,7 +824,6 @@ impl TransitionError { #[cfg(test)] pub(in crate::db::datastore) mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::test::{ sled_baseboard_for_test, sled_system_hardware_for_test, }; @@ -834,6 +833,7 @@ pub(in crate::db::datastore) mod test { use crate::db::lookup::LookupPath; use crate::db::model::ByteCount; use crate::db::model::SqlU32; + use crate::db::pub_test_utils::TestDatabase; use anyhow::{Context, Result}; use itertools::Itertools; use nexus_db_model::Generation; diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index b332c57798..e0092459b5 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -19,7 +19,7 @@ use crate::db::model::{ SwitchPortAddressConfig, SwitchPortBgpPeerConfig, SwitchPortConfig, SwitchPortLinkConfig, SwitchPortRouteConfig, SwitchPortSettings, SwitchPortSettingsGroup, SwitchPortSettingsGroups, - SwitchVlanInterfaceConfig, + SwitchVlanInterfaceConfig, TxEqConfig, }; use crate::db::pagination::paginated; use crate::transaction_retry::OptionalError; @@ -102,6 +102,7 @@ pub struct SwitchPortSettingsCombinedResult { pub port: SwitchPortConfig, pub links: Vec, pub link_lldp: Vec, + pub tx_eq: Vec>, pub interfaces: Vec, pub vlan_interfaces: Vec, pub routes: Vec, @@ -117,6 +118,7 @@ impl SwitchPortSettingsCombinedResult { groups: Vec::new(), links: Vec::new(), link_lldp: Vec::new(), + tx_eq: Vec::new(), interfaces: Vec::new(), vlan_interfaces: Vec::new(), routes: Vec::new(), @@ -136,6 +138,11 @@ impl Into groups: self.groups.into_iter().map(Into::into).collect(), links: self.links.into_iter().map(Into::into).collect(), link_lldp: self.link_lldp.into_iter().map(Into::into).collect(), + tx_eq: self + .tx_eq + .into_iter() + .map(|t| if let Some(t) = t { Some(t.into()) } else { None }) + .collect(), interfaces: self.interfaces.into_iter().map(Into::into).collect(), vlan_interfaces: self .vlan_interfaces @@ -466,6 +473,31 @@ impl DataStore { .load_async::(&conn) .await?; + let tx_eq_ids_and_nulls :Vec>= result + .links + .iter() + .map(|link| link.tx_eq_config_id) + .collect(); + let tx_eq_ids: Vec = tx_eq_ids_and_nulls + .iter() + .cloned() + .flatten() + .collect(); + + use db::schema::tx_eq_config; + let configs = tx_eq_config::dsl::tx_eq_config + .filter(tx_eq_config::id.eq_any(tx_eq_ids)) + .select(TxEqConfig::as_select()) + .limit(1) + .load_async::(&conn) + .await?; + result.tx_eq = tx_eq_ids_and_nulls.iter().map(|x| + if let Some(id) = x { + configs.iter().find(|c| c.id == *id).cloned() + } else { + None + }).collect(); + // get the interface configs use db::schema::switch_port_settings_interface_config::{ self as interface_config, dsl as interface_config_dsl, @@ -887,12 +919,15 @@ impl DataStore { Ok(v) => Ok(Some(v)), Err(e) => { let msg = "failed to check if bgp peer exists in switch port settings"; - error!(opctx.log, "{msg}"; "error" => ?e); match e { diesel::result::Error::NotFound => { + debug!(opctx.log, "{msg}"; "error" => ?e); Ok(None) - } - _ => Err(err.bail(Error::internal_error(msg))), + }, + _ => { + error!(opctx.log, "{msg}"; "error" => ?e); + Err(err.bail(Error::internal_error(msg))) + } } } }?; @@ -1112,6 +1147,7 @@ async fn do_switch_port_settings_create( switch_port_settings_port_config::dsl as port_config_dsl, switch_port_settings_route_config::dsl as route_config_dsl, switch_vlan_interface_config::dsl as vlan_config_dsl, + tx_eq_config::dsl as tx_eq_config_dsl, }; // create the top level port settings object @@ -1146,6 +1182,7 @@ async fn do_switch_port_settings_create( port: db_port_config, links: Vec::new(), link_lldp: Vec::new(), + tx_eq: Vec::new(), interfaces: Vec::new(), vlan_interfaces: Vec::new(), routes: Vec::new(), @@ -1158,6 +1195,7 @@ async fn do_switch_port_settings_create( let mut lldp_config = Vec::with_capacity(params.links.len()); let mut link_config = Vec::with_capacity(params.links.len()); + let mut tx_eq_config = Vec::with_capacity(params.links.len()); for (link_name, c) in ¶ms.links { let lldp_link_config = LldpLinkConfig::new( @@ -1172,6 +1210,20 @@ async fn do_switch_port_settings_create( let lldp_config_id = lldp_link_config.id; lldp_config.push(lldp_link_config); + let tx_eq_config_id = match &c.tx_eq { + Some(t) => { + let config = + TxEqConfig::new(t.pre1, t.pre2, t.main, t.post2, t.post1); + let tx_eq_config_id = config.id; + tx_eq_config.push(Some(config)); + Some(tx_eq_config_id) + } + _ => { + tx_eq_config.push(None); + None + } + }; + link_config.push(SwitchPortLinkConfig::new( psid, lldp_config_id, @@ -1180,6 +1232,7 @@ async fn do_switch_port_settings_create( c.fec.into(), c.speed.into(), c.autoneg, + tx_eq_config_id, )); } result.link_lldp = @@ -1189,6 +1242,16 @@ async fn do_switch_port_settings_create( .get_results_async(conn) .await?; + // We want to insert the Some(config) values into the table, but preserve the + // full vector of None/Some values. + let v: Vec = tx_eq_config.iter().flatten().cloned().collect(); + let _ = diesel::insert_into(tx_eq_config_dsl::tx_eq_config) + .values(v) + .returning(TxEqConfig::as_returning()) + .get_results_async(conn) + .await?; + result.tx_eq = tx_eq_config; + result.links = diesel::insert_into(link_config_dsl::switch_port_settings_link_config) .values(link_config) @@ -1517,6 +1580,15 @@ async fn do_switch_port_settings_delete( .execute_async(conn) .await?; + // delete tx_eq configs + use db::schema::tx_eq_config; + let tx_eq_ids: Vec = + links.iter().filter_map(|link| link.tx_eq_config_id).collect(); + diesel::delete(tx_eq_config::dsl::tx_eq_config) + .filter(tx_eq_config::id.eq_any(tx_eq_ids)) + .execute_async(conn) + .await?; + // delete interface configs use db::schema::switch_port_settings_interface_config::{ self as sps_interface_config, dsl as interface_config_dsl, @@ -1619,8 +1691,8 @@ async fn do_switch_port_settings_delete( #[cfg(test)] mod test { - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::UpdatePrecondition; + use crate::db::pub_test_utils::TestDatabase; use nexus_types::external_api::params::{ BgpAnnounceSetCreate, BgpConfigCreate, BgpPeerConfig, SwitchPortConfigCreate, SwitchPortGeometry, SwitchPortSettingsCreate, diff --git a/nexus/db-queries/src/db/datastore/test_utils.rs b/nexus/db-queries/src/db/datastore/test_utils.rs index 75d8833873..c3e9137ac8 100644 --- a/nexus/db-queries/src/db/datastore/test_utils.rs +++ b/nexus/db-queries/src/db/datastore/test_utils.rs @@ -17,21 +17,9 @@ use futures::future::try_join_all; use nexus_db_model::SledState; use nexus_types::external_api::views::SledPolicy; use nexus_types::external_api::views::SledProvisionPolicy; -use omicron_test_utils::dev::db::CockroachInstance; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::SledUuid; -use slog::Logger; -use std::sync::Arc; use strum::EnumCount; -use uuid::Uuid; - -pub(crate) async fn datastore_test( - log: &Logger, - db: &CockroachInstance, -) -> (OpContext, Arc) { - let rack_id = Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap(); - super::pub_test_utils::datastore_test(log, db, rack_id).await -} /// Denotes a specific way in which a sled is ineligible. /// diff --git a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs index a72c032125..d7639205de 100644 --- a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs +++ b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs @@ -326,8 +326,8 @@ impl DataStore { mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::lookup::LookupPath; + use crate::db::pub_test_utils::TestDatabase; use nexus_db_model::Instance; use nexus_db_model::Project; use nexus_db_model::SiloQuotasUpdate; diff --git a/nexus/db-queries/src/db/datastore/vmm.rs b/nexus/db-queries/src/db/datastore/vmm.rs index e578bb1696..15d8404fd6 100644 --- a/nexus/db-queries/src/db/datastore/vmm.rs +++ b/nexus/db-queries/src/db/datastore/vmm.rs @@ -441,11 +441,11 @@ impl DataStore { mod tests { use super::*; use crate::db; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::model::Generation; use crate::db::model::Migration; use crate::db::model::VmmRuntimeState; use crate::db::model::VmmState; + use crate::db::pub_test_utils::TestDatabase; use omicron_common::api::internal::nexus; use omicron_test_utils::dev; use omicron_uuid_kinds::InstanceUuid; diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index 93ba737eb5..88b40fcf64 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -2780,7 +2780,7 @@ impl DataStore { mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use omicron_test_utils::dev; use sled_agent_client::types::CrucibleOpts; diff --git a/nexus/db-queries/src/db/datastore/volume_repair.rs b/nexus/db-queries/src/db/datastore/volume_repair.rs index 115244f347..7ea88c8542 100644 --- a/nexus/db-queries/src/db/datastore/volume_repair.rs +++ b/nexus/db-queries/src/db/datastore/volume_repair.rs @@ -100,7 +100,7 @@ impl DataStore { mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use omicron_test_utils::dev; #[tokio::test] diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index e3bd33e0a4..30033e96a2 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -2755,11 +2755,11 @@ impl DataStore { #[cfg(test)] mod tests { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::test::sled_baseboard_for_test; use crate::db::datastore::test::sled_system_hardware_for_test; use crate::db::datastore::test_utils::IneligibleSleds; use crate::db::model::Project; + use crate::db::pub_test_utils::TestDatabase; use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; use nexus_db_fixed_data::silo::DEFAULT_SILO; use nexus_db_fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; @@ -2792,7 +2792,6 @@ mod tests { // `project_create_vpc_raw` call. #[tokio::test] async fn test_project_create_vpc_raw_returns_none_on_vni_exhaustion() { - usdt::register_probes().unwrap(); let logctx = dev::test_setup_log( "test_project_create_vpc_raw_returns_none_on_vni_exhaustion", ); @@ -2898,7 +2897,6 @@ mod tests { // and then check that we correctly retry #[tokio::test] async fn test_project_create_vpc_retries() { - usdt::register_probes().unwrap(); let logctx = dev::test_setup_log("test_project_create_vpc_retries"); let log = &logctx.log; let db = TestDatabase::new_with_datastore(log).await; @@ -3042,7 +3040,6 @@ mod tests { #[tokio::test] async fn test_vpc_resolve_to_sleds_uses_current_target_blueprint() { // Test setup. - usdt::register_probes().unwrap(); let logctx = dev::test_setup_log( "test_vpc_resolve_to_sleds_uses_current_target_blueprint", ); @@ -3404,7 +3401,6 @@ mod tests { // and that these resolve to the v4/6 subnets of each. #[tokio::test] async fn test_vpc_system_router_sync_to_subnets() { - usdt::register_probes().unwrap(); let logctx = dev::test_setup_log("test_vpc_system_router_sync_to_subnets"); let log = &logctx.log; @@ -3631,7 +3627,6 @@ mod tests { // of an instance NIC. #[tokio::test] async fn test_vpc_router_rule_instance_resolve() { - usdt::register_probes().unwrap(); let logctx = dev::test_setup_log("test_vpc_router_rule_instance_resolve"); let log = &logctx.log; diff --git a/nexus/db-queries/src/db/explain.rs b/nexus/db-queries/src/db/explain.rs index 284e96bc6e..6ff74737d0 100644 --- a/nexus/db-queries/src/db/explain.rs +++ b/nexus/db-queries/src/db/explain.rs @@ -94,7 +94,7 @@ mod test { use super::*; use crate::db; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::AsyncSimpleConnection; use diesel::SelectableHelper; use expectorate::assert_contents; diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index 43cd2a073f..4dce6eeeab 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -958,8 +958,8 @@ mod test { use super::Instance; use super::LookupPath; use super::Project; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::model::Name; + use crate::db::pub_test_utils::TestDatabase; use omicron_test_utils::dev; /* This is a smoke test that things basically appear to work. */ diff --git a/nexus/db-queries/src/db/mod.rs b/nexus/db-queries/src/db/mod.rs index fc44a2f27b..880e653591 100644 --- a/nexus/db-queries/src/db/mod.rs +++ b/nexus/db-queries/src/db/mod.rs @@ -36,6 +36,8 @@ mod update_and_check; // full table scans the same way pooled connections do. pub use pool_connection::DISALLOW_FULL_TABLE_SCAN_SQL; +#[cfg(any(test, feature = "testing"))] +pub mod pub_test_utils; pub mod test_utils; pub use nexus_db_fixed_data as fixed_data; diff --git a/nexus/db-queries/src/db/on_conflict_ext.rs b/nexus/db-queries/src/db/on_conflict_ext.rs index 25e05d1b0f..5f31eb99fb 100644 --- a/nexus/db-queries/src/db/on_conflict_ext.rs +++ b/nexus/db-queries/src/db/on_conflict_ext.rs @@ -127,8 +127,8 @@ pub trait IncompleteOnConflictExt { /// _logically implies_ the condition on the partial index. With something /// like `time_deleted IS NULL` the value of that is not exactly clear, but /// you can imagine a partial index on something like `col >= 10`, and - /// write `ON CONFLICT (...) WHERE col >= 20`. This is allowed because `col - /// >= 20` implies `col >= 10`. (But `WHERE col >= 5` is not allowed.) + /// write `ON CONFLICT (...) WHERE col >= 20`. This is allowed because + /// `col >= 20` implies `col >= 10`. (But `WHERE col >= 5` is not allowed.) /// /// ## 4. A similar syntax with a different meaning /// diff --git a/nexus/db-queries/src/db/pagination.rs b/nexus/db-queries/src/db/pagination.rs index a16591ad6c..01911eb802 100644 --- a/nexus/db-queries/src/db/pagination.rs +++ b/nexus/db-queries/src/db/pagination.rs @@ -343,7 +343,7 @@ mod test { use super::*; use crate::db; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use diesel::JoinOnDsl; use diesel::SelectableHelper; diff --git a/nexus/db-queries/src/db/pool.rs b/nexus/db-queries/src/db/pool.rs index c42158a64f..ab032e022a 100644 --- a/nexus/db-queries/src/db/pool.rs +++ b/nexus/db-queries/src/db/pool.rs @@ -84,17 +84,20 @@ impl Pool { /// Creating this pool does not necessarily wait for connections to become /// available, as backends may shift over time. pub fn new(log: &Logger, resolver: &QorbResolver) -> Self { - // Make sure diesel-dtrace's USDT probes are enabled. - usdt::register_probes().expect("Failed to register USDT DTrace probes"); - let resolver = resolver.for_service(ServiceName::Cockroach); let connector = make_postgres_connector(log); - let policy = Policy::default(); - Pool { - inner: qorb::pool::Pool::new(resolver, connector, policy), - terminated: std::sync::atomic::AtomicBool::new(false), - } + let inner = match qorb::pool::Pool::new(resolver, connector, policy) { + Ok(pool) => { + debug!(log, "registered USDT probes"); + pool + } + Err(err) => { + error!(log, "failed to register USDT probes"); + err.into_inner() + } + }; + Pool { inner, terminated: std::sync::atomic::AtomicBool::new(false) } } /// Creates a new qorb-backed connection pool to a single instance of the @@ -105,17 +108,20 @@ impl Pool { /// /// In production, [Self::new] should be preferred. pub fn new_single_host(log: &Logger, db_config: &DbConfig) -> Self { - // Make sure diesel-dtrace's USDT probes are enabled. - usdt::register_probes().expect("Failed to register USDT DTrace probes"); - let resolver = make_single_host_resolver(db_config); let connector = make_postgres_connector(log); - let policy = Policy::default(); - Pool { - inner: qorb::pool::Pool::new(resolver, connector, policy), - terminated: std::sync::atomic::AtomicBool::new(false), - } + let inner = match qorb::pool::Pool::new(resolver, connector, policy) { + Ok(pool) => { + debug!(log, "registered USDT probes"); + pool + } + Err(err) => { + error!(log, "failed to register USDT probes"); + err.into_inner() + } + }; + Pool { inner, terminated: std::sync::atomic::AtomicBool::new(false) } } /// Creates a new qorb-backed connection pool which returns an error @@ -129,20 +135,23 @@ impl Pool { log: &Logger, db_config: &DbConfig, ) -> Self { - // Make sure diesel-dtrace's USDT probes are enabled. - usdt::register_probes().expect("Failed to register USDT DTrace probes"); - let resolver = make_single_host_resolver(db_config); let connector = make_postgres_connector(log); - let policy = Policy { claim_timeout: tokio::time::Duration::from_millis(1), ..Default::default() }; - Pool { - inner: qorb::pool::Pool::new(resolver, connector, policy), - terminated: std::sync::atomic::AtomicBool::new(false), - } + let inner = match qorb::pool::Pool::new(resolver, connector, policy) { + Ok(pool) => { + debug!(log, "registered USDT probes"); + pool + } + Err(err) => { + error!(log, "failed to register USDT probes"); + err.into_inner() + } + }; + Pool { inner, terminated: std::sync::atomic::AtomicBool::new(false) } } /// Returns a connection from the pool diff --git a/nexus/db-queries/src/db/pool_connection.rs b/nexus/db-queries/src/db/pool_connection.rs index 9a33370a5a..84fb975b2d 100644 --- a/nexus/db-queries/src/db/pool_connection.rs +++ b/nexus/db-queries/src/db/pool_connection.rs @@ -92,14 +92,13 @@ impl backend::Connector for DieselPgConnector { }) .await .expect("Task panicked establishing connection") - .map_err(|e| { + .inspect_err(|e| { warn!( self.log, "Failed to make connection"; "error" => e.to_string(), "backend" => backend.address, ); - e })?; Ok(conn) } diff --git a/nexus/db-queries/src/db/pub_test_utils/crdb.rs b/nexus/db-queries/src/db/pub_test_utils/crdb.rs new file mode 100644 index 0000000000..aa06f0681e --- /dev/null +++ b/nexus/db-queries/src/db/pub_test_utils/crdb.rs @@ -0,0 +1,62 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Database testing facilities. + +use camino::Utf8PathBuf; +use omicron_test_utils::dev; +use slog::Logger; + +/// Path to the "seed" CockroachDB tarball. +/// +/// Populating CockroachDB unfortunately isn't free - creation of +/// tables, indices, and users takes several seconds to complete. +/// +/// By creating a "seed" version of the database, we can cut down +/// on the time spent performing this operation. Instead, we opt +/// to copy the database from this seed location. +fn seed_tar() -> Utf8PathBuf { + // The setup script should set this environment variable. + let seed_dir = std::env::var(dev::CRDB_SEED_TAR_ENV).unwrap_or_else(|_| { + panic!( + "{} missing -- are you running this test \ + with `cargo nextest run`?", + dev::CRDB_SEED_TAR_ENV, + ) + }); + seed_dir.into() +} + +/// Wrapper around [`dev::test_setup_database`] which uses a seed tarball +/// provided from the environment. +pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance { + let input_tar = seed_tar(); + dev::test_setup_database( + log, + dev::StorageSource::CopyFromSeed { input_tar }, + ) + .await +} + +/// Creates a new database with no data populated. +/// +/// Primarily used for schema change and migration testing. +pub async fn test_setup_database_empty( + log: &Logger, +) -> dev::db::CockroachInstance { + dev::test_setup_database(log, dev::StorageSource::DoNotPopulate).await +} + +/// Wrapper around [`dev::test_setup_database`] which uses a seed tarball +/// provided as an argument. +pub async fn test_setup_database_from_seed( + log: &Logger, + input_tar: Utf8PathBuf, +) -> dev::db::CockroachInstance { + dev::test_setup_database( + log, + dev::StorageSource::CopyFromSeed { input_tar }, + ) + .await +} diff --git a/nexus/db-queries/src/db/pub_test_utils/mod.rs b/nexus/db-queries/src/db/pub_test_utils/mod.rs new file mode 100644 index 0000000000..3e18453803 --- /dev/null +++ b/nexus/db-queries/src/db/pub_test_utils/mod.rs @@ -0,0 +1,330 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Test support code that can be enabled by dependencies via this crate's +//! `testing` feature. +//! +//! This feature should only be enabled under `dev-dependencies` to avoid this +//! test support code leaking into release binaries. + +use crate::authz; +use crate::context::OpContext; +use crate::db; +use crate::db::DataStore; +use omicron_test_utils::dev::db::CockroachInstance; +use slog::Logger; +use std::sync::Arc; +use uuid::Uuid; + +pub mod crdb; + +enum Populate { + Nothing, + Schema, + SchemaAndData, +} + +enum Interface { + Nothing, + Pool, + Datastore, +} + +fn new_pool(log: &Logger, db: &CockroachInstance) -> Arc { + let cfg = db::Config { url: db.pg_config().clone() }; + Arc::new(db::Pool::new_single_host(log, &cfg)) +} + +struct TestDatabaseBuilder { + populate: Populate, + interface: Interface, +} + +impl TestDatabaseBuilder { + /// Creates a new database buidler. + /// + /// By default, this creates a database with no schema, and with no pools + /// nor datastores built on top. + /// + /// This is equivalent to calling [Self::populate_nothing] and + /// [Self::no_interface]. + fn new() -> Self { + Self { populate: Populate::Nothing, interface: Interface::Nothing } + } + + /// Populates the database without a schema + fn populate_nothing(self) -> Self { + self.populate(Populate::Nothing) + } + + /// Populates the database with a schema + fn populate_schema(self) -> Self { + self.populate(Populate::Schema) + } + + /// Populates the database with a schema and loads it with data + fn populate_schema_and_builtin_data(self) -> Self { + self.populate(Populate::SchemaAndData) + } + + /// Builds no interface on top of the database (neither pool nor datastore) + fn no_interface(self) -> Self { + self.interface(Interface::Nothing) + } + + /// Builds a pool interface on top of the database + fn interface_pool(self) -> Self { + self.interface(Interface::Pool) + } + + /// Builds a datatore interface on top of the database + fn interface_datastore(self) -> Self { + self.interface(Interface::Datastore) + } + + async fn build(self, log: &Logger) -> TestDatabase { + match (self.populate, self.interface) { + (Populate::Nothing, interface) => { + let db = crdb::test_setup_database_empty(log).await; + match interface { + Interface::Nothing => { + TestDatabase { db, kind: TestKind::NoPool } + } + Interface::Pool => { + let pool = new_pool(log, &db); + TestDatabase { db, kind: TestKind::Pool { pool } } + } + Interface::Datastore => { + panic!("Cannot create datastore without schema") + } + } + } + (Populate::Schema, interface) => { + let db = crdb::test_setup_database(log).await; + match interface { + Interface::Nothing => { + TestDatabase { db, kind: TestKind::NoPool } + } + Interface::Pool => { + let pool = new_pool(log, &db); + TestDatabase { db, kind: TestKind::Pool { pool } } + } + Interface::Datastore => { + let pool = new_pool(log, &db); + let datastore = Arc::new( + DataStore::new(&log, pool, None).await.unwrap(), + ); + TestDatabase { + db, + kind: TestKind::RawDatastore { datastore }, + } + } + } + } + (Populate::SchemaAndData, Interface::Datastore) => { + let db = crdb::test_setup_database(log).await; + let (opctx, datastore) = + datastore_test_on_default_rack(log, &db).await; + TestDatabase { + db, + kind: TestKind::Datastore { opctx, datastore }, + } + } + (Populate::SchemaAndData, Interface::Nothing) + | (Populate::SchemaAndData, Interface::Pool) => { + // This configuration isn't wrong, it's just weird - we need to + // build a datastore to load the built-in data, so it's odd to + // discard it immediately afterwards. + panic!("If you're fully populating a datastore, you probably want a connection to it"); + } + } + } + + fn populate(self, populate: Populate) -> Self { + Self { populate, ..self } + } + + fn interface(self, interface: Interface) -> Self { + Self { interface, ..self } + } +} + +enum TestKind { + NoPool, + Pool { pool: Arc }, + RawDatastore { datastore: Arc }, + Datastore { opctx: OpContext, datastore: Arc }, +} + +/// A test database, possibly with a pool or full datastore on top +pub struct TestDatabase { + db: CockroachInstance, + kind: TestKind, +} + +impl TestDatabase { + /// Creates a new database for test usage, without any schema nor interface + /// + /// [`Self::terminate`] should be called before the test finishes. + pub async fn new_populate_nothing(log: &Logger) -> Self { + TestDatabaseBuilder::new() + .populate_nothing() + .no_interface() + .build(log) + .await + } + + /// Creates a new database for test usage, with a schema but no interface + /// + /// [`Self::terminate`] should be called before the test finishes. + pub async fn new_populate_schema_only(log: &Logger) -> Self { + TestDatabaseBuilder::new() + .populate_schema() + .no_interface() + .build(log) + .await + } + + /// Creates a new database for test usage, with a pool. + /// + /// [`Self::terminate`] should be called before the test finishes. + pub async fn new_with_pool(log: &Logger) -> Self { + TestDatabaseBuilder::new() + .populate_schema() + .interface_pool() + .build(log) + .await + } + + /// Creates a new database for test usage, with a pre-loaded datastore. + /// + /// [`Self::terminate`] should be called before the test finishes. + pub async fn new_with_datastore(log: &Logger) -> Self { + TestDatabaseBuilder::new() + .populate_schema_and_builtin_data() + .interface_datastore() + .build(log) + .await + } + + /// Creates a new database for test usage, with a schema but no builtin data + /// + /// [`Self::terminate`] should be called before the test finishes. + pub async fn new_with_raw_datastore(log: &Logger) -> Self { + TestDatabaseBuilder::new() + .populate_schema() + .interface_datastore() + .build(log) + .await + } + + pub fn crdb(&self) -> &CockroachInstance { + &self.db + } + + pub fn pool(&self) -> &Arc { + match &self.kind { + TestKind::Pool { pool } => pool, + TestKind::NoPool + | TestKind::RawDatastore { .. } + | TestKind::Datastore { .. } => { + panic!( + "Wrong test type; try using `TestDatabase::new_with_pool`" + ); + } + } + } + + pub fn opctx(&self) -> &OpContext { + match &self.kind { + TestKind::NoPool + | TestKind::Pool { .. } + | TestKind::RawDatastore { .. } => { + panic!("Wrong test type; try using `TestDatabase::new_with_datastore`"); + } + TestKind::Datastore { opctx, .. } => opctx, + } + } + + pub fn datastore(&self) -> &Arc { + match &self.kind { + TestKind::NoPool | TestKind::Pool { .. } => { + panic!("Wrong test type; try using `TestDatabase::new_with_datastore`"); + } + TestKind::RawDatastore { datastore } => datastore, + TestKind::Datastore { datastore, .. } => datastore, + } + } + + /// Shuts down both the database and the pool + pub async fn terminate(mut self) { + match self.kind { + TestKind::NoPool => (), + TestKind::Pool { pool } => pool.terminate().await, + TestKind::RawDatastore { datastore } => datastore.terminate().await, + TestKind::Datastore { datastore, .. } => { + datastore.terminate().await + } + } + self.db.cleanup().await.unwrap(); + } +} + +pub const RACK_UUID: &str = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"; + +async fn datastore_test_on_default_rack( + log: &Logger, + db: &CockroachInstance, +) -> (OpContext, Arc) { + let rack_id = Uuid::parse_str(RACK_UUID).unwrap(); + datastore_test(log, db, rack_id).await +} + +// Constructs a DataStore for use in test suites that has preloaded the +// built-in users, roles, and role assignments that are needed for basic +// operation +async fn datastore_test( + log: &Logger, + db: &CockroachInstance, + rack_id: Uuid, +) -> (OpContext, Arc) { + use crate::authn; + + let cfg = db::Config { url: db.pg_config().clone() }; + let pool = Arc::new(db::Pool::new_single_host(&log, &cfg)); + let datastore = Arc::new(DataStore::new(&log, pool, None).await.unwrap()); + + // Create an OpContext with the credentials of "db-init" just for the + // purpose of loading the built-in users, roles, and assignments. + let opctx = OpContext::for_background( + log.new(o!()), + Arc::new(authz::Authz::new(&log)), + authn::Context::internal_db_init(), + Arc::clone(&datastore) as Arc, + ); + + // TODO: Can we just call "Populate" instead of doing this? + datastore.load_builtin_users(&opctx).await.unwrap(); + datastore.load_builtin_roles(&opctx).await.unwrap(); + datastore.load_builtin_role_asgns(&opctx).await.unwrap(); + datastore.load_builtin_silos(&opctx).await.unwrap(); + datastore.load_builtin_projects(&opctx).await.unwrap(); + datastore.load_builtin_vpcs(&opctx).await.unwrap(); + datastore.load_silo_users(&opctx).await.unwrap(); + datastore.load_silo_user_role_assignments(&opctx).await.unwrap(); + datastore + .load_builtin_fleet_virtual_provisioning_collection(&opctx) + .await + .unwrap(); + datastore.load_builtin_rack_data(&opctx, rack_id).await.unwrap(); + + // Create an OpContext with the credentials of "test-privileged" for general + // testing. + let opctx = OpContext::for_tests( + log.new(o!()), + Arc::clone(&datastore) as Arc, + ); + + (opctx, datastore) +} diff --git a/nexus/db-queries/src/db/queries/external_ip.rs b/nexus/db-queries/src/db/queries/external_ip.rs index d1028fbdb6..8cc33e96d9 100644 --- a/nexus/db-queries/src/db/queries/external_ip.rs +++ b/nexus/db-queries/src/db/queries/external_ip.rs @@ -865,13 +865,13 @@ impl RunQueryDsl for NextExternalIp {} #[cfg(test)] mod tests { use crate::authz; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::SERVICE_IP_POOL_NAME; use crate::db::identity::Resource; use crate::db::lookup::LookupPath; use crate::db::model::IpKind; use crate::db::model::IpPool; use crate::db::model::IpPoolRange; + use crate::db::pub_test_utils::TestDatabase; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use dropshot::test_util::LogContext; diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs index 39c799d223..6f955edf73 100644 --- a/nexus/db-queries/src/db/queries/network_interface.rs +++ b/nexus/db-queries/src/db/queries/network_interface.rs @@ -1797,7 +1797,6 @@ mod tests { use super::NUM_INITIAL_RESERVED_IP_ADDRESSES; use crate::authz; use crate::context::OpContext; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::DataStore; use crate::db::identity::Resource; use crate::db::lookup::LookupPath; @@ -1808,6 +1807,7 @@ mod tests { use crate::db::model::NetworkInterface; use crate::db::model::Project; use crate::db::model::VpcSubnet; + use crate::db::pub_test_utils::TestDatabase; use crate::db::queries::network_interface::last_available_address; use async_bb8_diesel::AsyncRunQueryDsl; use dropshot::test_util::LogContext; diff --git a/nexus/db-queries/src/db/queries/next_item.rs b/nexus/db-queries/src/db/queries/next_item.rs index 0ec0727737..be9cc7a715 100644 --- a/nexus/db-queries/src/db/queries/next_item.rs +++ b/nexus/db-queries/src/db/queries/next_item.rs @@ -924,8 +924,8 @@ mod tests { use super::NextItem; use super::ShiftIndices; use crate::db; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::explain::ExplainableAsync as _; + use crate::db::pub_test_utils::TestDatabase; use crate::db::queries::next_item::NextItemSelfJoined; use async_bb8_diesel::AsyncRunQueryDsl; use async_bb8_diesel::AsyncSimpleConnection; diff --git a/nexus/db-queries/src/db/queries/oximeter.rs b/nexus/db-queries/src/db/queries/oximeter.rs index eee7dd5669..3279daa6a2 100644 --- a/nexus/db-queries/src/db/queries/oximeter.rs +++ b/nexus/db-queries/src/db/queries/oximeter.rs @@ -221,8 +221,8 @@ pub fn reassign_producers_query(oximeter_id: Uuid) -> TypedSqlQuery<()> { #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::explain::ExplainableAsync; + use crate::db::pub_test_utils::TestDatabase; use crate::db::raw_query_builder::expectorate_query_contents; use omicron_test_utils::dev; use std::time::Duration; diff --git a/nexus/db-queries/src/db/queries/region_allocation.rs b/nexus/db-queries/src/db/queries/region_allocation.rs index a531e726b3..efdc9a21fb 100644 --- a/nexus/db-queries/src/db/queries/region_allocation.rs +++ b/nexus/db-queries/src/db/queries/region_allocation.rs @@ -405,9 +405,9 @@ UNION #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::datastore::REGION_REDUNDANCY_THRESHOLD; use crate::db::explain::ExplainableAsync; + use crate::db::pub_test_utils::TestDatabase; use crate::db::raw_query_builder::expectorate_query_contents; use omicron_test_utils::dev; use uuid::Uuid; diff --git a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs index 8d3ef320ee..6fa0a46e53 100644 --- a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs +++ b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs @@ -478,8 +478,8 @@ FROM #[cfg(test)] mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::explain::ExplainableAsync; + use crate::db::pub_test_utils::TestDatabase; use crate::db::raw_query_builder::expectorate_query_contents; use omicron_test_utils::dev; use uuid::Uuid; diff --git a/nexus/db-queries/src/db/queries/vpc_subnet.rs b/nexus/db-queries/src/db/queries/vpc_subnet.rs index 17ee5103aa..54c03f592e 100644 --- a/nexus/db-queries/src/db/queries/vpc_subnet.rs +++ b/nexus/db-queries/src/db/queries/vpc_subnet.rs @@ -288,9 +288,9 @@ impl InsertVpcSubnetError { mod test { use super::InsertVpcSubnetError; use super::InsertVpcSubnetQuery; - use crate::db::datastore::pub_test_utils::TestDatabase; use crate::db::explain::ExplainableAsync as _; use crate::db::model::VpcSubnet; + use crate::db::pub_test_utils::TestDatabase; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::Name; use omicron_test_utils::dev; diff --git a/nexus/db-queries/src/policy_test/mod.rs b/nexus/db-queries/src/policy_test/mod.rs index be81f58138..0b44560631 100644 --- a/nexus/db-queries/src/policy_test/mod.rs +++ b/nexus/db-queries/src/policy_test/mod.rs @@ -14,7 +14,7 @@ mod coverage; mod resource_builder; mod resources; -use crate::db::datastore::pub_test_utils::TestDatabase; +use crate::db::pub_test_utils::TestDatabase; use coverage::Coverage; use futures::StreamExt; use nexus_auth::authn; diff --git a/nexus/db-queries/src/transaction_retry.rs b/nexus/db-queries/src/transaction_retry.rs index ccd1ec4ecd..02d00f8215 100644 --- a/nexus/db-queries/src/transaction_retry.rs +++ b/nexus/db-queries/src/transaction_retry.rs @@ -246,7 +246,7 @@ impl OptionalError { mod test { use super::*; - use crate::db::datastore::pub_test_utils::TestDatabase; + use crate::db::pub_test_utils::TestDatabase; use omicron_test_utils::dev; use oximeter::types::FieldValue; diff --git a/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql b/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql index 4e7dde244b..9ee71b403f 100644 --- a/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql @@ -271,7 +271,10 @@ WITH dataset.port, dataset.kind, dataset.size_used, - dataset.zone_name + dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression ) ( SELECT @@ -286,6 +289,9 @@ WITH dataset.kind, dataset.size_used, dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -313,6 +319,9 @@ UNION updated_datasets.kind, updated_datasets.size_used, updated_datasets.zone_name, + updated_datasets.quota, + updated_datasets.reservation, + updated_datasets.compression, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/db-queries/tests/output/region_allocate_random_sleds.sql b/nexus/db-queries/tests/output/region_allocate_random_sleds.sql index b2c164a6d9..369410c68c 100644 --- a/nexus/db-queries/tests/output/region_allocate_random_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_random_sleds.sql @@ -269,7 +269,10 @@ WITH dataset.port, dataset.kind, dataset.size_used, - dataset.zone_name + dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression ) ( SELECT @@ -284,6 +287,9 @@ WITH dataset.kind, dataset.size_used, dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -311,6 +317,9 @@ UNION updated_datasets.kind, updated_datasets.size_used, updated_datasets.zone_name, + updated_datasets.quota, + updated_datasets.reservation, + updated_datasets.compression, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql b/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql index 97ee23f82e..9251139c4e 100644 --- a/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql @@ -282,7 +282,10 @@ WITH dataset.port, dataset.kind, dataset.size_used, - dataset.zone_name + dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression ) ( SELECT @@ -297,6 +300,9 @@ WITH dataset.kind, dataset.size_used, dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -324,6 +330,9 @@ UNION updated_datasets.kind, updated_datasets.size_used, updated_datasets.zone_name, + updated_datasets.quota, + updated_datasets.reservation, + updated_datasets.compression, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql b/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql index a1cc103594..c8aa8adf2e 100644 --- a/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql @@ -280,7 +280,10 @@ WITH dataset.port, dataset.kind, dataset.size_used, - dataset.zone_name + dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression ) ( SELECT @@ -295,6 +298,9 @@ WITH dataset.kind, dataset.size_used, dataset.zone_name, + dataset.quota, + dataset.reservation, + dataset.compression, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -322,6 +328,9 @@ UNION updated_datasets.kind, updated_datasets.size_used, updated_datasets.zone_name, + updated_datasets.quota, + updated_datasets.reservation, + updated_datasets.compression, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/metrics-producer-gc/src/lib.rs b/nexus/metrics-producer-gc/src/lib.rs index 32e2be5809..27ef73f75a 100644 --- a/nexus/metrics-producer-gc/src/lib.rs +++ b/nexus/metrics-producer-gc/src/lib.rs @@ -184,8 +184,7 @@ mod tests { use httptest::responders::status_code; use httptest::Expectation; use nexus_db_model::OximeterInfo; - use nexus_db_queries::db::datastore::pub_test_utils::datastore_test; - use nexus_test_utils::db::test_setup_database; + use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_types::internal_api::params; use omicron_common::api::internal::nexus; use omicron_test_utils::dev; @@ -216,9 +215,8 @@ mod tests { async fn test_prune_expired_producers() { // Setup let logctx = dev::test_setup_log("test_prune_expired_producers"); - let mut db = test_setup_database(&logctx.log).await; - let (opctx, datastore) = - datastore_test(&logctx.log, &db, Uuid::new_v4()).await; + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); // Insert an Oximeter collector let collector_info = OximeterInfo::new(¶ms::OximeterInfo { @@ -291,8 +289,7 @@ mod tests { assert!(pruned.successes.is_empty()); assert!(pruned.failures.is_empty()); - datastore.terminate().await; - db.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } @@ -302,9 +299,8 @@ mod tests { let logctx = dev::test_setup_log( "test_prune_expired_producers_notifies_collector", ); - let mut db = test_setup_database(&logctx.log).await; - let (opctx, datastore) = - datastore_test(&logctx.log, &db, Uuid::new_v4()).await; + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); let mut collector = httptest::Server::run(); @@ -358,8 +354,7 @@ mod tests { collector.verify_and_clear(); - datastore.terminate().await; - db.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } } diff --git a/nexus/reconfigurator/execution/src/datasets.rs b/nexus/reconfigurator/execution/src/datasets.rs index cf53a24a8f..5d3bb30a37 100644 --- a/nexus/reconfigurator/execution/src/datasets.rs +++ b/nexus/reconfigurator/execution/src/datasets.rs @@ -4,36 +4,136 @@ //! Ensures dataset records required by a given blueprint +use crate::Sled; + +use anyhow::anyhow; use anyhow::Context; +use futures::stream; +use futures::StreamExt; use nexus_db_model::Dataset; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; -use nexus_types::deployment::BlueprintZoneConfig; -use nexus_types::deployment::DurableDataset; +use nexus_types::deployment::BlueprintDatasetConfig; +use nexus_types::deployment::BlueprintDatasetDisposition; +use nexus_types::deployment::BlueprintDatasetsConfig; use nexus_types::identity::Asset; +use omicron_common::disk::DatasetConfig; +use omicron_common::disk::DatasetsConfig; +use omicron_uuid_kinds::BlueprintUuid; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; -use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::SledUuid; use slog::info; +use slog::o; use slog::warn; -use std::collections::BTreeSet; +use std::collections::BTreeMap; + +/// Idempotently ensures that the specified datasets are deployed to the +/// corresponding sleds +pub(crate) async fn deploy_datasets( + opctx: &OpContext, + sleds_by_id: &BTreeMap, + sled_configs: &BTreeMap, +) -> Result<(), Vec> { + let errors: Vec<_> = stream::iter(sled_configs) + .filter_map(|(sled_id, config)| async move { + let log = opctx.log.new(o!( + "sled_id" => sled_id.to_string(), + "generation" => config.generation.to_string(), + )); + + let db_sled = match sleds_by_id.get(&sled_id) { + Some(sled) => sled, + None => { + let err = anyhow!("sled not found in db list: {}", sled_id); + warn!(log, "{err:#}"); + return Some(err); + } + }; + + let client = nexus_networking::sled_client_from_address( + sled_id.into_untyped_uuid(), + db_sled.sled_agent_address(), + &log, + ); + + let config: DatasetsConfig = config.clone().into(); + let result = + client.datasets_put(&config).await.with_context( + || format!("Failed to put {config:#?} to sled {sled_id}"), + ); + match result { + Err(error) => { + warn!(log, "{error:#}"); + Some(error) + } + Ok(result) => { + let (errs, successes): (Vec<_>, Vec<_>) = result + .into_inner() + .status + .into_iter() + .partition(|status| status.err.is_some()); + + if !errs.is_empty() { + warn!( + log, + "Failed to deploy datasets for sled agent"; + "successfully configured datasets" => successes.len(), + "failed dataset configurations" => errs.len(), + ); + for err in &errs { + warn!(log, "{err:?}"); + } + return Some(anyhow!( + "failure deploying datasets: {:?}", + errs + )); + } + + info!( + log, + "Successfully deployed datasets for sled agent"; + "successfully configured datasets" => successes.len(), + ); + None + } + } + }) + .collect() + .await; + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } +} + +#[allow(dead_code)] +pub(crate) struct EnsureDatasetsResult { + pub(crate) inserted: usize, + pub(crate) updated: usize, + pub(crate) removed: usize, +} -/// For each zone in `all_omicron_zones` that has an associated durable dataset, -/// ensure that a corresponding dataset record exists in `datastore`. +/// For all datasets we expect to see in the blueprint, ensure that a corresponding +/// database record exists in `datastore`. /// -/// Does not modify any existing dataset records. Returns the number of -/// datasets inserted. +/// Updates all existing dataset records that don't match the blueprint. +/// Returns the number of datasets changed. pub(crate) async fn ensure_dataset_records_exist( opctx: &OpContext, datastore: &DataStore, - all_omicron_zones: impl Iterator, -) -> anyhow::Result { + bp_id: BlueprintUuid, + bp_datasets: impl Iterator, +) -> anyhow::Result { // Before attempting to insert any datasets, first query for any existing // dataset records so we can filter them out. This looks like a typical // TOCTOU issue, but it is purely a performance optimization. We expect // almost all executions of this function to do nothing: new datasets are // created very rarely relative to how frequently blueprint realization // happens. We could remove this check and filter and instead run the below - // "insert if not exists" query on every zone, and the behavior would still + // "insert if not exists" query on every dataset, and the behavior would still // be correct. However, that would issue far more queries than necessary in // the very common case of "we don't need to do anything at all". let mut existing_datasets = datastore @@ -41,60 +141,90 @@ pub(crate) async fn ensure_dataset_records_exist( .await .context("failed to list all datasets")? .into_iter() - .map(|dataset| OmicronZoneUuid::from_untyped_uuid(dataset.id())) - .collect::>(); + .map(|dataset| (DatasetUuid::from_untyped_uuid(dataset.id()), dataset)) + .collect::>(); let mut num_inserted = 0; - let mut num_already_exist = 0; + let mut num_updated = 0; + let mut num_unchanged = 0; + let mut num_removed = 0; - for zone in all_omicron_zones { - let Some(DurableDataset { dataset, kind, address }) = - zone.zone_type.durable_dataset() - else { - continue; - }; + let (wanted_datasets, unwanted_datasets): (Vec<_>, Vec<_>) = bp_datasets + .partition(|d| match d.disposition { + BlueprintDatasetDisposition::InService => true, + BlueprintDatasetDisposition::Expunged => false, + }); - let id = zone.id; + for bp_dataset in wanted_datasets { + let id = bp_dataset.id; + let kind = &bp_dataset.kind; - // If already present in the datastore, move on. - if existing_datasets.remove(&id) { - num_already_exist += 1; - continue; - } + // If this dataset already exists, only update it if it appears different from what exists + // in the database already. + let action = if let Some(db_dataset) = existing_datasets.remove(&id) { + let db_config: DatasetConfig = db_dataset.try_into()?; + let bp_config: DatasetConfig = bp_dataset.clone().try_into()?; - let pool_id = dataset.pool_name.id(); - let dataset = Dataset::new( - id.into_untyped_uuid(), - pool_id.into_untyped_uuid(), - Some(address), - kind.clone(), - ); - let maybe_inserted = datastore - .dataset_insert_if_not_exists(dataset) + if db_config == bp_config { + num_unchanged += 1; + continue; + } + num_updated += 1; + "update" + } else { + num_inserted += 1; + "insert" + }; + + let dataset = Dataset::from(bp_dataset.clone()); + datastore + .dataset_upsert_if_blueprint_is_current_target( + &opctx, bp_id, dataset, + ) .await .with_context(|| { - format!("failed to insert dataset record for dataset {id}") + format!("failed to upsert dataset record for dataset {id}") })?; - // If we succeeded in inserting, log it; if `maybe_dataset` is `None`, - // we must have lost the TOCTOU race described above, and another Nexus - // must have inserted this dataset before we could. - if maybe_inserted.is_some() { - info!( - opctx.log, - "inserted new dataset for Omicron zone"; - "id" => %id, - "kind" => ?kind, - ); - num_inserted += 1; - } else { - num_already_exist += 1; + info!( + opctx.log, + "ensuring dataset record in database"; + "action" => action, + "id" => %id, + "kind" => ?kind, + ); + } + + for bp_dataset in unwanted_datasets { + if existing_datasets.remove(&bp_dataset.id).is_some() { + if matches!( + bp_dataset.kind, + omicron_common::disk::DatasetKind::Crucible + ) { + // Region and snapshot replacement cannot happen without the + // database record, even if the dataset has been expunged. + // + // This record will still be deleted, but it will happen as a + // part of the "decommissioned_disk_cleaner" background task. + continue; + } + + datastore + .dataset_delete_if_blueprint_is_current_target( + &opctx, + bp_id, + bp_dataset.id, + ) + .await?; + num_removed += 1; } } - // We don't currently support removing datasets, so this would be - // surprising: the database contains dataset records that are no longer in - // our blueprint. We can't do anything about this, so just warn. + // We support removing expunged datasets - if we read a dataset that hasn't + // been explicitly expunged, log this as an oddity. + // + // This could be possible in conditions where multiple Nexuses are executing + // distinct blueprints. if !existing_datasets.is_empty() { warn!( opctx.log, @@ -106,12 +236,18 @@ pub(crate) async fn ensure_dataset_records_exist( info!( opctx.log, - "ensured all Omicron zones have dataset records"; + "ensured all Omicron datasets have database records"; "num_inserted" => num_inserted, - "num_already_existed" => num_already_exist, + "num_updated" => num_updated, + "num_unchanged" => num_unchanged, + "num_removed" => num_removed, ); - Ok(num_inserted) + Ok(EnsureDatasetsResult { + inserted: num_inserted, + updated: num_updated, + removed: num_removed, + }) } #[cfg(test)] @@ -119,12 +255,12 @@ mod tests { use super::*; use nexus_db_model::Zpool; use nexus_reconfigurator_planning::example::ExampleSystemBuilder; - use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_test_utils_macros::nexus_test; - use nexus_types::deployment::blueprint_zone_type; - use nexus_types::deployment::BlueprintZoneDisposition; + use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintZoneFilter; - use nexus_types::deployment::BlueprintZoneType; + use omicron_common::api::external::ByteCount; + use omicron_common::api::internal::shared::DatasetKind; + use omicron_common::disk::CompressionAlgorithm; use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::ZpoolUuid; @@ -133,11 +269,30 @@ mod tests { type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; + fn get_all_datasets_from_zones( + blueprint: &Blueprint, + ) -> Vec { + blueprint + .all_omicron_zones(BlueprintZoneFilter::All) + .filter_map(|(_, zone)| { + let dataset = zone.zone_type.durable_dataset()?; + Some(BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: DatasetUuid::new_v4(), + pool: dataset.dataset.pool_name.clone(), + kind: dataset.kind, + address: Some(dataset.address), + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, + }) + }) + .collect::>() + } + #[nexus_test] - async fn test_ensure_dataset_records_exist( - cptestctx: &ControlPlaneTestContext, - ) { - const TEST_NAME: &str = "test_ensure_dataset_records_exist"; + async fn test_dataset_record_create(cptestctx: &ControlPlaneTestContext) { + const TEST_NAME: &str = "test_dataset_record_create"; // Set up. let nexus = &cptestctx.server.server_context().nexus; @@ -149,8 +304,12 @@ mod tests { let opctx = &opctx; // Use the standard example system. - let (example, blueprint) = + let (example, mut blueprint) = ExampleSystemBuilder::new(&opctx.log, TEST_NAME).nsleds(5).build(); + + // Set the target so our database-modifying operations know they + // can safely act on the current target blueprint. + update_blueprint_target(&datastore, &opctx, &mut blueprint).await; let collection = example.collection; // Record the sleds and zpools. @@ -170,29 +329,32 @@ mod tests { 0 ); - // Collect all the blueprint zones. - let all_omicron_zones = blueprint - .all_omicron_zones(BlueprintZoneFilter::All) - .map(|(_, zone)| zone) - .collect::>(); + // Let's allocate datasets for all the zones with durable datasets. + // + // Finding these datasets is normally the responsibility of the planner, + // but we're kinda hand-rolling it. + let all_datasets = get_all_datasets_from_zones(&blueprint); // How many zones are there with durable datasets? - let nzones_with_durable_datasets = all_omicron_zones - .iter() - .filter(|z| z.zone_type.durable_dataset().is_some()) - .count(); + let nzones_with_durable_datasets = all_datasets.len(); + assert!(nzones_with_durable_datasets > 0); - let ndatasets_inserted = ensure_dataset_records_exist( - opctx, - datastore, - all_omicron_zones.iter().copied(), - ) - .await - .expect("failed to ensure datasets"); + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); // We should have inserted a dataset for each zone with a durable // dataset. - assert_eq!(nzones_with_durable_datasets, ndatasets_inserted); + assert_eq!(inserted, nzones_with_durable_datasets); + assert_eq!(updated, 0); + assert_eq!(removed, 0); assert_eq!( datastore .dataset_list_all_batched(opctx, None) @@ -203,14 +365,18 @@ mod tests { ); // Ensuring the same datasets again should insert no new records. - let ndatasets_inserted = ensure_dataset_records_exist( - opctx, - datastore, - all_omicron_zones.iter().copied(), - ) - .await - .expect("failed to ensure datasets"); - assert_eq!(0, ndatasets_inserted); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 0); + assert_eq!(updated, 0); + assert_eq!(removed, 0); assert_eq!( datastore .dataset_list_all_batched(opctx, None) @@ -235,44 +401,44 @@ mod tests { .expect("failed to upsert zpool"); } - // Call `ensure_dataset_records_exist` again, adding new crucible and - // cockroach zones. It should insert only these new zones. + // Call `ensure_dataset_records_exist` again, adding new datasets. + // + // It should only insert these new zones. let new_zones = [ - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: OmicronZoneUuid::new_v4(), - filesystem_pool: Some(ZpoolName::new_external(new_zpool_id)), - zone_type: BlueprintZoneType::Crucible( - blueprint_zone_type::Crucible { - address: "[::1]:0".parse().unwrap(), - dataset: OmicronZoneDataset { - pool_name: ZpoolName::new_external(new_zpool_id), - }, - }, - ), + BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: DatasetUuid::new_v4(), + pool: ZpoolName::new_external(new_zpool_id), + kind: DatasetKind::Debug, + address: None, + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, }, - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: OmicronZoneUuid::new_v4(), - filesystem_pool: Some(ZpoolName::new_external(new_zpool_id)), - zone_type: BlueprintZoneType::CockroachDb( - blueprint_zone_type::CockroachDb { - address: "[::1]:0".parse().unwrap(), - dataset: OmicronZoneDataset { - pool_name: ZpoolName::new_external(new_zpool_id), - }, - }, - ), + BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: DatasetUuid::new_v4(), + pool: ZpoolName::new_external(new_zpool_id), + kind: DatasetKind::TransientZoneRoot, + address: None, + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, }, ]; - let ndatasets_inserted = ensure_dataset_records_exist( - opctx, - datastore, - all_omicron_zones.iter().copied().chain(&new_zones), - ) - .await - .expect("failed to ensure datasets"); - assert_eq!(ndatasets_inserted, 2); + + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter().chain(&new_zones), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 2); + assert_eq!(updated, 0); + assert_eq!(removed, 0); assert_eq!( datastore .dataset_list_all_batched(opctx, None) @@ -282,4 +448,320 @@ mod tests { nzones_with_durable_datasets + 2, ); } + + // Sets the target blueprint to "blueprint" + // + // Reads the current target, and uses it as the "parent" blueprint + async fn update_blueprint_target( + datastore: &DataStore, + opctx: &OpContext, + blueprint: &mut Blueprint, + ) { + // Fetch the initial blueprint installed during rack initialization. + let parent_blueprint_target = datastore + .blueprint_target_get_current(&opctx) + .await + .expect("failed to read current target blueprint"); + blueprint.parent_blueprint_id = Some(parent_blueprint_target.target_id); + datastore.blueprint_insert(&opctx, &blueprint).await.unwrap(); + datastore + .blueprint_target_set_current( + &opctx, + nexus_types::deployment::BlueprintTarget { + target_id: blueprint.id, + enabled: true, + time_made_target: nexus_inventory::now_db_precision(), + }, + ) + .await + .unwrap(); + } + + #[nexus_test] + async fn test_dataset_records_update(cptestctx: &ControlPlaneTestContext) { + const TEST_NAME: &str = "test_dataset_records_update"; + + // Set up. + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + let opctx = &opctx; + + // Use the standard example system. + let (_example, mut blueprint) = + ExampleSystemBuilder::new(&opctx.log, TEST_NAME).nsleds(5).build(); + + // Set the target so our database-modifying operations know they + // can safely act on the current target blueprint. + update_blueprint_target(&datastore, &opctx, &mut blueprint).await; + + // Record the sleds and zpools. + crate::tests::insert_sled_records(datastore, &blueprint).await; + crate::tests::create_disks_for_zones_using_datasets( + datastore, opctx, &blueprint, + ) + .await; + + let mut all_datasets = get_all_datasets_from_zones(&blueprint); + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, all_datasets.len()); + assert_eq!(updated, 0); + assert_eq!(removed, 0); + + // These values don't *really* matter, we just want to make sure we can + // change them and see the update. + let first_dataset = &mut all_datasets[0]; + assert_eq!(first_dataset.quota, None); + assert_eq!(first_dataset.reservation, None); + assert_eq!(first_dataset.compression, CompressionAlgorithm::Off); + + first_dataset.quota = Some(ByteCount::from_kibibytes_u32(1)); + first_dataset.reservation = Some(ByteCount::from_kibibytes_u32(2)); + first_dataset.compression = CompressionAlgorithm::Lz4; + + // Update the datastore + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 0); + assert_eq!(updated, 1); + assert_eq!(removed, 0); + + // Observe that the update stuck + let observed_datasets = + datastore.dataset_list_all_batched(opctx, None).await.unwrap(); + let first_dataset = &mut all_datasets[0]; + let observed_dataset = observed_datasets + .into_iter() + .find(|dataset| { + dataset.id() == first_dataset.id.into_untyped_uuid() + }) + .expect("Couldn't find dataset we tried to update?"); + let observed_dataset: DatasetConfig = + observed_dataset.try_into().unwrap(); + assert_eq!(observed_dataset.quota, first_dataset.quota); + assert_eq!(observed_dataset.reservation, first_dataset.reservation); + assert_eq!(observed_dataset.compression, first_dataset.compression); + } + + #[nexus_test] + async fn test_dataset_records_delete(cptestctx: &ControlPlaneTestContext) { + const TEST_NAME: &str = "test_dataset_records_delete"; + + // Set up. + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + let opctx = &opctx; + + // Use the standard example system. + let (_example, mut blueprint) = + ExampleSystemBuilder::new(&opctx.log, TEST_NAME).nsleds(5).build(); + + // Set the target so our database-modifying operations know they + // can safely act on the current target blueprint. + update_blueprint_target(&datastore, &opctx, &mut blueprint).await; + + // Record the sleds and zpools. + crate::tests::insert_sled_records(datastore, &blueprint).await; + crate::tests::create_disks_for_zones_using_datasets( + datastore, opctx, &blueprint, + ) + .await; + + let mut all_datasets = get_all_datasets_from_zones(&blueprint); + + // Ensure that a non-crucible dataset exists + all_datasets.push(BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: DatasetUuid::new_v4(), + pool: all_datasets[0].pool.clone(), + kind: DatasetKind::Debug, + address: None, + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, + }); + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, all_datasets.len()); + assert_eq!(updated, 0); + assert_eq!(removed, 0); + + // Expunge two datasets -- one for Crucible, and one for any other + // service. + + let crucible_dataset = all_datasets + .iter_mut() + .find(|dataset| matches!(dataset.kind, DatasetKind::Crucible)) + .expect("No crucible dataset found"); + assert_eq!( + crucible_dataset.disposition, + BlueprintDatasetDisposition::InService + ); + crucible_dataset.disposition = BlueprintDatasetDisposition::Expunged; + let crucible_dataset_id = crucible_dataset.id; + + let non_crucible_dataset = all_datasets + .iter_mut() + .find(|dataset| !matches!(dataset.kind, DatasetKind::Crucible)) + .expect("No non-crucible dataset found"); + assert_eq!( + non_crucible_dataset.disposition, + BlueprintDatasetDisposition::InService + ); + non_crucible_dataset.disposition = + BlueprintDatasetDisposition::Expunged; + let non_crucible_dataset_id = non_crucible_dataset.id; + + // Observe that we only remove one dataset. + // + // This is a property of "special-case handling" of the Crucible + // dataset, where we punt the deletion to a background task. + + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 0); + assert_eq!(updated, 0); + assert_eq!(removed, 1); + + // Make sure the Crucible dataset still exists, even if the other + // dataset got deleted. + + let observed_datasets = + datastore.dataset_list_all_batched(opctx, None).await.unwrap(); + assert!(observed_datasets + .iter() + .any(|d| d.id() == crucible_dataset_id.into_untyped_uuid())); + assert!(!observed_datasets + .iter() + .any(|d| d.id() == non_crucible_dataset_id.into_untyped_uuid())); + } + + #[nexus_test] + async fn test_dataset_record_blueprint_removal_without_expunging( + cptestctx: &ControlPlaneTestContext, + ) { + const TEST_NAME: &str = + "test_dataset_record_blueprint_removal_without_expunging"; + + // Set up. + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + let opctx = &opctx; + + // Use the standard example system. + let (_example, mut blueprint) = + ExampleSystemBuilder::new(&opctx.log, TEST_NAME).nsleds(5).build(); + + // Set the target so our database-modifying operations know they + // can safely act on the current target blueprint. + update_blueprint_target(&datastore, &opctx, &mut blueprint).await; + + // Record the sleds and zpools. + crate::tests::insert_sled_records(datastore, &blueprint).await; + crate::tests::create_disks_for_zones_using_datasets( + datastore, opctx, &blueprint, + ) + .await; + + let mut all_datasets = get_all_datasets_from_zones(&blueprint); + + // Ensure that a deletable dataset exists + let dataset_id = DatasetUuid::new_v4(); + all_datasets.push(BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: dataset_id, + pool: all_datasets[0].pool.clone(), + kind: DatasetKind::Debug, + address: None, + quota: None, + reservation: None, + compression: CompressionAlgorithm::Off, + }); + + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, all_datasets.len()); + assert_eq!(updated, 0); + assert_eq!(removed, 0); + + // Rather than expunging a dataset, which is the normal way to "delete" + // a dataset, we'll just remove it from the "blueprint". + // + // This situation mimics a scenario where we are an "old Nexus, + // executing an old blueprint" - more datasets might be created + // concurrently with our execution, and we should leave them alone. + assert_eq!(dataset_id, all_datasets.pop().unwrap().id); + + // Observe that no datasets are removed. + let EnsureDatasetsResult { inserted, updated, removed } = + ensure_dataset_records_exist( + opctx, + datastore, + bp_id, + all_datasets.iter(), + ) + .await + .expect("failed to ensure datasets"); + assert_eq!(inserted, 0); + assert_eq!(updated, 0); + assert_eq!(removed, 0); + + // Make sure the dataset still exists, even if it isn't tracked by our + // "blueprint". + let observed_datasets = + datastore.dataset_list_all_batched(opctx, None).await.unwrap(); + assert!(observed_datasets + .iter() + .any(|d| d.id() == dataset_id.into_untyped_uuid())); + } } diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index cd14069d50..e0d3414f1b 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -661,6 +661,7 @@ mod test { id: Uuid::new_v4(), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state, cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, @@ -1336,6 +1337,8 @@ mod test { .unwrap(); let zpool_rows = datastore.zpool_list_all_external_batched(&opctx).await.unwrap(); + let dataset_rows = + datastore.dataset_list_all_batched(&opctx, None).await.unwrap(); let ip_pool_range_rows = { let (authz_service_ip_pool, _) = datastore.ip_pools_service_lookup(&opctx).await.unwrap(); @@ -1348,6 +1351,7 @@ mod test { let mut builder = PlanningInputFromDb { sled_rows: &sled_rows, zpool_rows: &zpool_rows, + dataset_rows: &dataset_rows, ip_pool_range_rows: &ip_pool_range_rows, internal_dns_version: dns_initial_internal.generation.into(), external_dns_version: dns_latest_external.generation.into(), @@ -1395,7 +1399,15 @@ mod test { let rv = builder .sled_ensure_zone_multiple_nexus(sled_id, nalready + 1) .unwrap(); - assert_eq!(rv, EnsureMultiple::Changed { added: 1, removed: 0 }); + assert_eq!( + rv, + EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 0 + } + ); let blueprint2 = builder.build(); eprintln!("blueprint2: {}", blueprint2.display()); // Figure out the id of the new zone. diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs index e160ddc9a0..48ab3996ff 100644 --- a/nexus/reconfigurator/execution/src/lib.rs +++ b/nexus/reconfigurator/execution/src/lib.rs @@ -12,11 +12,13 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::execution::*; use nexus_types::deployment::Blueprint; +use nexus_types::deployment::BlueprintDatasetFilter; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::SledFilter; use nexus_types::external_api::views::SledState; use nexus_types::identity::Asset; use omicron_physical_disks::DeployDisksDone; +use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; @@ -115,6 +117,13 @@ pub async fn realize_blueprint_with_overrides( sled_list.clone(), ); + register_deploy_datasets_step( + &engine.for_component(ExecutionComponent::Datasets), + &opctx, + blueprint, + sled_list.clone(), + ); + register_deploy_zones_step( &engine.for_component(ExecutionComponent::OmicronZones), &opctx, @@ -283,6 +292,32 @@ fn register_deploy_disks_step<'a>( .register() } +fn register_deploy_datasets_step<'a>( + registrar: &ComponentRegistrar<'_, 'a>, + opctx: &'a OpContext, + blueprint: &'a Blueprint, + sleds: SharedStepHandle>>, +) { + registrar + .new_step( + ExecutionStepId::Ensure, + "Deploy datasets", + move |cx| async move { + let sleds_by_id = sleds.into_value(cx.token()).await; + datasets::deploy_datasets( + &opctx, + &sleds_by_id, + &blueprint.blueprint_datasets, + ) + .await + .map_err(merge_anyhow_list)?; + + StepSuccess::new(()).into() + }, + ) + .register(); +} + fn register_deploy_zones_step<'a>( registrar: &ComponentRegistrar<'_, 'a>, opctx: &'a OpContext, @@ -348,6 +383,7 @@ fn register_dataset_records_step<'a>( datastore: &'a DataStore, blueprint: &'a Blueprint, ) { + let bp_id = BlueprintUuid::from_untyped_uuid(blueprint.id); registrar .new_step( ExecutionStepId::Ensure, @@ -356,9 +392,8 @@ fn register_dataset_records_step<'a>( datasets::ensure_dataset_records_exist( &opctx, datastore, - blueprint - .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) - .map(|(_sled_id, zone)| zone), + bp_id, + blueprint.all_omicron_datasets(BlueprintDatasetFilter::All), ) .await?; diff --git a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs index 895b7a2d9d..b56c9c0433 100644 --- a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs +++ b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs @@ -66,7 +66,7 @@ pub(crate) async fn deploy_disks( if !errs.is_empty() { warn!( log, - "Failed to deploy storage for sled agent"; + "Failed to deploy physical disk for sled agent"; "successfully configured disks" => successes.len(), "failed disk configurations" => errs.len(), ); @@ -81,7 +81,7 @@ pub(crate) async fn deploy_disks( info!( log, - "Successfully deployed storage for sled agent"; + "Successfully deployed physical disks for sled agent"; "successfully configured disks" => successes.len(), ); None @@ -181,6 +181,7 @@ mod test { id, blueprint_zones: BTreeMap::new(), blueprint_disks, + blueprint_datasets: BTreeMap::new(), sled_state: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs index 0e299cebd3..b7587377c9 100644 --- a/nexus/reconfigurator/execution/src/omicron_zones.rs +++ b/nexus/reconfigurator/execution/src/omicron_zones.rs @@ -383,6 +383,7 @@ mod test { id, blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml index 9607e26394..19e429dcd9 100644 --- a/nexus/reconfigurator/planning/Cargo.toml +++ b/nexus/reconfigurator/planning/Cargo.toml @@ -12,6 +12,7 @@ clickhouse-admin-types.workspace = true chrono.workspace = true debug-ignore.workspace = true gateway-client.workspace = true +illumos-utils.workspace = true indexmap.workspace = true internal-dns-resolver.workspace = true ipnet.workspace = true diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index fb7d1f2dca..bf4977a8b2 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -5,6 +5,7 @@ //! Low-level facility for generating Blueprints use crate::ip_allocator::IpAllocator; +use crate::planner::rng::PlannerRng; use crate::planner::zone_needs_expungement; use crate::planner::ZoneExpungeReason; use anyhow::anyhow; @@ -15,6 +16,9 @@ use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_sled_agent_shared::inventory::ZoneKind; use nexus_types::deployment::blueprint_zone_type; use nexus_types::deployment::Blueprint; +use nexus_types::deployment::BlueprintDatasetConfig; +use nexus_types::deployment::BlueprintDatasetDisposition; +use nexus_types::deployment::BlueprintDatasetsConfig; use nexus_types::deployment::BlueprintPhysicalDiskConfig; use nexus_types::deployment::BlueprintPhysicalDisksConfig; use nexus_types::deployment::BlueprintZoneConfig; @@ -45,21 +49,23 @@ use omicron_common::address::DNS_HTTP_PORT; use omicron_common::address::DNS_PORT; use omicron_common::address::NTP_PORT; use omicron_common::address::SLED_RESERVED_ADDRESSES; +use omicron_common::api::external::ByteCount; use omicron_common::api::external::Generation; use omicron_common::api::external::Vni; +use omicron_common::api::internal::shared::DatasetKind; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; +use omicron_common::disk::CompressionAlgorithm; +use omicron_common::disk::DatasetConfig; +use omicron_common::disk::DatasetName; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; -use omicron_uuid_kinds::ExternalIpKind; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::GenericUuid; -use omicron_uuid_kinds::OmicronZoneKind; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use once_cell::unsync::OnceCell; -use rand::rngs::StdRng; -use rand::SeedableRng; use slog::debug; use slog::error; use slog::info; @@ -75,8 +81,6 @@ use std::net::Ipv6Addr; use std::net::SocketAddr; use std::net::SocketAddrV6; use thiserror::Error; -use typed_rng::TypedUuidRng; -use typed_rng::UuidRng; use super::clickhouse::ClickhouseAllocator; use super::external_networking::BuilderExternalNetworking; @@ -140,8 +144,19 @@ pub enum Ensure { /// actions taken or no action was necessary #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum EnsureMultiple { - /// action was taken, and multiple items were added - Changed { added: usize, removed: usize }, + /// action was taken within the operation + Changed { + /// An item was added to the blueprint + added: usize, + /// An item was updated within the blueprint + updated: usize, + /// An item was expunged in the blueprint + expunged: usize, + /// An item was removed from the blueprint. + /// + /// This usually happens after the work of expungment has completed. + removed: usize, + }, /// no action was necessary NotNeeded, @@ -154,9 +169,28 @@ pub enum EnsureMultiple { /// "comment", identifying which operations have occurred on the blueprint. #[derive(Debug, Clone, Eq, PartialEq)] pub(crate) enum Operation { - AddZone { sled_id: SledUuid, kind: ZoneKind }, - UpdateDisks { sled_id: SledUuid, added: usize, removed: usize }, - ZoneExpunged { sled_id: SledUuid, reason: ZoneExpungeReason, count: usize }, + AddZone { + sled_id: SledUuid, + kind: ZoneKind, + }, + UpdateDisks { + sled_id: SledUuid, + added: usize, + updated: usize, + removed: usize, + }, + UpdateDatasets { + sled_id: SledUuid, + added: usize, + updated: usize, + expunged: usize, + removed: usize, + }, + ZoneExpunged { + sled_id: SledUuid, + reason: ZoneExpungeReason, + count: usize, + }, } impl fmt::Display for Operation { @@ -165,8 +199,17 @@ impl fmt::Display for Operation { Self::AddZone { sled_id, kind } => { write!(f, "sled {sled_id}: added zone: {}", kind.report_str()) } - Self::UpdateDisks { sled_id, added, removed } => { - write!(f, "sled {sled_id}: added {added} disks, removed {removed} disks") + Self::UpdateDisks { sled_id, added, updated, removed } => { + write!(f, "sled {sled_id}: added {added} disks, updated {updated}, removed {removed} disks") + } + Self::UpdateDatasets { + sled_id, + added, + updated, + expunged, + removed, + } => { + write!(f, "sled {sled_id}: added {added} datasets, updated: {updated}, expunged {expunged}, removed {removed} datasets") } Self::ZoneExpunged { sled_id, reason, count } => { let reason = match reason { @@ -195,6 +238,13 @@ impl fmt::Display for Operation { } } +fn zone_name(zone: &BlueprintZoneConfig) -> String { + illumos_utils::zone::zone_name( + zone.zone_type.kind().zone_prefix(), + Some(zone.id), + ) +} + /// Helper for assembling a blueprint /// /// There are two basic ways to assemble a new blueprint: @@ -232,6 +282,7 @@ pub struct BlueprintBuilder<'a> { // corresponding fields in `Blueprint`. pub(super) zones: BlueprintZonesBuilder<'a>, disks: BlueprintDisksBuilder<'a>, + datasets: BlueprintDatasetsBuilder<'a>, sled_state: BTreeMap, cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade, @@ -240,7 +291,7 @@ pub struct BlueprintBuilder<'a> { comments: Vec, // Random number generator for new UUIDs - rng: BlueprintBuilderRng, + rng: PlannerRng, } impl<'a> BlueprintBuilder<'a> { @@ -250,11 +301,7 @@ impl<'a> BlueprintBuilder<'a> { sled_ids: impl Iterator, creator: &str, ) -> Blueprint { - Self::build_empty_with_sleds_impl( - sled_ids, - creator, - BlueprintBuilderRng::new(), - ) + Self::build_empty_with_sleds_impl(sled_ids, creator, PlannerRng::new()) } /// A version of [`Self::build_empty_with_sleds`] that allows the @@ -264,7 +311,7 @@ impl<'a> BlueprintBuilder<'a> { creator: &str, seed: H, ) -> Blueprint { - let mut rng = BlueprintBuilderRng::new(); + let mut rng = PlannerRng::new(); rng.set_seed(seed); Self::build_empty_with_sleds_impl(sled_ids, creator, rng) } @@ -272,7 +319,7 @@ impl<'a> BlueprintBuilder<'a> { fn build_empty_with_sleds_impl( sled_ids: impl Iterator, creator: &str, - mut rng: BlueprintBuilderRng, + mut rng: PlannerRng, ) -> Blueprint { let blueprint_zones = sled_ids .map(|sled_id| { @@ -291,9 +338,10 @@ impl<'a> BlueprintBuilder<'a> { .collect(); Blueprint { - id: rng.blueprint_rng.next(), + id: rng.next_blueprint(), blueprint_zones, blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state, parent_blueprint_id: None, internal_dns_version: Generation::new(), @@ -394,6 +442,7 @@ impl<'a> BlueprintBuilder<'a> { internal_dns_subnets: OnceCell::new(), zones: BlueprintZonesBuilder::new(parent_blueprint), disks: BlueprintDisksBuilder::new(parent_blueprint), + datasets: BlueprintDatasetsBuilder::new(parent_blueprint), sled_state, cockroachdb_setting_preserve_downgrade: parent_blueprint .cockroachdb_setting_preserve_downgrade, @@ -401,7 +450,7 @@ impl<'a> BlueprintBuilder<'a> { creator: creator.to_owned(), operations: Vec::new(), comments: Vec::new(), - rng: BlueprintBuilderRng::new(), + rng: PlannerRng::new(), }) } @@ -478,6 +527,9 @@ impl<'a> BlueprintBuilder<'a> { let blueprint_disks = self .disks .into_disks_map(self.input.all_sled_ids(SledFilter::InService)); + let blueprint_datasets = self + .datasets + .into_datasets_map(self.input.all_sled_ids(SledFilter::InService)); // If we have an allocator, use it to generate a new config. If an error // is returned then log it and carry over the parent_config. @@ -491,9 +543,10 @@ impl<'a> BlueprintBuilder<'a> { } }); Blueprint { - id: self.rng.blueprint_rng.next(), + id: self.rng.next_blueprint(), blueprint_zones, blueprint_disks, + blueprint_datasets, sled_state: self.sled_state, parent_blueprint_id: Some(self.parent_blueprint.id), internal_dns_version: self.input.internal_dns_version(), @@ -778,7 +831,208 @@ impl<'a> BlueprintBuilder<'a> { !removals.contains(&PhysicalDiskUuid::from_untyped_uuid(config.id)) }); - Ok(EnsureMultiple::Changed { added, removed }) + Ok(EnsureMultiple::Changed { added, updated: 0, expunged: 0, removed }) + } + + /// Ensures that a sled in the blueprint has all the datasets it should. + /// + /// We perform the following process to decide what datasets should exist + /// in the blueprint during the planning phase: + /// + /// INPUT | OUTPUT + /// ---------------------------------------------------------------------- + /// zpools in the blueprint | blueprint datasets for debug, root filesystem + /// | (All zpools should have these datasets) + /// ---------------------------------------------------------------------- + /// zones in the blueprint | blueprint datasets for filesystems, durable data + /// | (These datasets are needed for zones) + /// ---------------------------------------------------------------------- + /// discretionary datasets | blueprint datasets for discretionary datasets + /// NOTE: These don't exist, | + /// at the moment | + /// ---------------------------------------------------------------------- + /// + /// From this process, we should be able to construct "all datasets that + /// should exist in the new blueprint". + /// + /// - If new datasets are proposed, they are added to the blueprint. + /// - If datasets are changed, they are updated in the blueprint. + /// - If datasets are not proposed, but they exist in the parent blueprint, + /// they are expunged. + pub fn sled_ensure_datasets( + &mut self, + sled_id: SledUuid, + resources: &SledResources, + ) -> Result { + const DEBUG_QUOTA_SIZE_GB: u32 = 100; + + let (mut additions, mut updates, mut expunges, removals) = { + let mut datasets_builder = BlueprintSledDatasetsBuilder::new( + self.log.clone(), + sled_id, + &self.datasets, + resources, + ); + + // Ensure each zpool has a "Debug" and "Zone Root" dataset. + let bp_zpools = self + .disks + .current_sled_disks(sled_id) + .map(|disk_config| disk_config.pool_id) + .collect::>(); + for zpool_id in bp_zpools { + let zpool = ZpoolName::new_external(zpool_id); + let address = None; + datasets_builder.ensure( + DatasetName::new(zpool.clone(), DatasetKind::Debug), + address, + Some(ByteCount::from_gibibytes_u32(DEBUG_QUOTA_SIZE_GB)), + None, + CompressionAlgorithm::Off, + ); + datasets_builder.ensure( + DatasetName::new(zpool, DatasetKind::TransientZoneRoot), + address, + None, + None, + CompressionAlgorithm::Off, + ); + } + + // Ensure that datasets needed for zones exist. + for (zone, _zone_state) in self.zones.current_sled_zones( + sled_id, + BlueprintZoneFilter::ShouldBeRunning, + ) { + // Dataset for transient zone filesystem + if let Some(fs_zpool) = &zone.filesystem_pool { + let name = zone_name(&zone); + let address = None; + datasets_builder.ensure( + DatasetName::new( + fs_zpool.clone(), + DatasetKind::TransientZone { name }, + ), + address, + None, + None, + CompressionAlgorithm::Off, + ); + } + + // Dataset for durable dataset co-located with zone + if let Some(dataset) = zone.zone_type.durable_dataset() { + let zpool = &dataset.dataset.pool_name; + let address = match zone.zone_type { + BlueprintZoneType::Crucible( + blueprint_zone_type::Crucible { address, .. }, + ) => Some(address), + _ => None, + }; + datasets_builder.ensure( + DatasetName::new(zpool.clone(), dataset.kind), + address, + None, + None, + CompressionAlgorithm::Off, + ); + } + } + + // TODO: Note that we also have datasets in "zone/" for propolis + // zones, but these are not currently being tracked by blueprints. + + let expunges = datasets_builder.get_expungeable_datasets(); + let removals = datasets_builder.get_removable_datasets(); + + let additions = datasets_builder + .new_datasets + .into_values() + .flat_map(|datasets| datasets.into_values().map(|d| (d.id, d))) + .collect::>(); + let updates = datasets_builder + .updated_datasets + .into_values() + .flat_map(|datasets| { + datasets.into_values().map(|dataset| (dataset.id, dataset)) + }) + .collect::>(); + (additions, updates, expunges, removals) + }; + + if additions.is_empty() + && updates.is_empty() + && expunges.is_empty() + && removals.is_empty() + { + return Ok(EnsureMultiple::NotNeeded); + } + let added = additions.len(); + let updated = updates.len(); + // - When a dataset is expunged, for whatever reason, it is a part of + // "expunges". This leads to it getting removed from a sled. + // - When we know that we've safely destroyed all traces of the dataset, + // it becomes a part of "removals". This means we can remove it from the + // blueprint. + let expunged = expunges.len(); + let removed = removals.len(); + + let datasets = + &mut self.datasets.change_sled_datasets(sled_id).datasets; + + // Add all new datasets + datasets.append(&mut additions); + + for config in datasets.values_mut() { + // Apply updates + if let Some(new_config) = updates.remove(&config.id) { + *config = new_config; + }; + + // Mark unused datasets as expunged. + // + // This indicates that the dataset should be removed from the database. + if expunges.remove(&config.id) { + config.disposition = BlueprintDatasetDisposition::Expunged; + } + + // Small optimization -- if no expungement nor updates are left, + // bail + if expunges.is_empty() && updates.is_empty() { + break; + } + } + + // These conditions should be dead-code, and arguably could be + // assertions, but are safety nets to catch programming errors. + if !expunges.is_empty() { + return Err(Error::Planner(anyhow!( + "Should have marked all expunged datasets" + ))); + } + if !updates.is_empty() { + return Err(Error::Planner(anyhow!( + "Should have applied all updates" + ))); + } + + // Remove all datasets that we've finished expunging. + datasets.retain(|_id, d| { + if removals.contains(&d.id) { + debug_assert_eq!( + d.disposition, + BlueprintDatasetDisposition::Expunged, + "Should only remove datasets that are expunged, but dataset {} is {:?}", + d.id, d.disposition, + ); + return false; + }; + true + }); + + // We sort in the call to "BlueprintDatasetsBuilder::into_datasets_map", + // so we don't need to sort "datasets" now. + Ok(EnsureMultiple::Changed { added, updated, expunged, removed }) } fn sled_add_zone_internal_dns( @@ -802,7 +1056,7 @@ impl<'a> BlueprintBuilder<'a> { let zone = BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, - id: self.rng.zone_rng.next(), + id: self.rng.next_zone(), filesystem_pool: Some(zpool), zone_type, }; @@ -840,14 +1094,19 @@ impl<'a> BlueprintBuilder<'a> { )?; } - Ok(EnsureMultiple::Changed { added: to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: to_add, + removed: 0, + expunged: 0, + updated: 0, + }) } fn sled_add_zone_external_dns( &mut self, sled_id: SledUuid, ) -> Result { - let id = self.rng.zone_rng.next(); + let id = self.rng.next_zone(); let ExternalNetworkingChoice { external_ip, nic_ip, @@ -855,7 +1114,7 @@ impl<'a> BlueprintBuilder<'a> { nic_mac, } = self.external_networking()?.for_new_external_dns()?; let nic = NetworkInterface { - id: self.rng.network_interface_rng.next(), + id: self.rng.next_network_interface(), kind: NetworkInterfaceKind::Service { id: id.into_untyped_uuid() }, name: format!("external-dns-{id}").parse().unwrap(), ip: nic_ip, @@ -871,7 +1130,7 @@ impl<'a> BlueprintBuilder<'a> { let http_address = SocketAddrV6::new(underlay_address, DNS_HTTP_PORT, 0, 0); let dns_address = OmicronZoneExternalFloatingAddr { - id: self.rng.external_ip_rng.next(), + id: self.rng.next_external_ip(), addr: SocketAddr::new(external_ip, DNS_PORT), }; let pool_name = @@ -930,7 +1189,12 @@ impl<'a> BlueprintBuilder<'a> { } } - Ok(EnsureMultiple::Changed { added, removed: 0 }) + Ok(EnsureMultiple::Changed { + added, + updated: 0, + removed: 0, + expunged: 0, + }) } pub fn sled_ensure_zone_ntp( @@ -958,7 +1222,7 @@ impl<'a> BlueprintBuilder<'a> { let zone = BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, - id: self.rng.zone_rng.next(), + id: self.rng.next_zone(), filesystem_pool: Some(filesystem_pool), zone_type, }; @@ -1014,7 +1278,7 @@ impl<'a> BlueprintBuilder<'a> { let zone = BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, - id: self.rng.zone_rng.next(), + id: self.rng.next_zone(), filesystem_pool: Some(filesystem_pool), zone_type, }; @@ -1098,7 +1362,7 @@ impl<'a> BlueprintBuilder<'a> { }; for _ in 0..num_nexus_to_add { - let nexus_id = self.rng.zone_rng.next(); + let nexus_id = self.rng.next_zone(); let ExternalNetworkingChoice { external_ip, nic_ip, @@ -1106,12 +1370,12 @@ impl<'a> BlueprintBuilder<'a> { nic_mac, } = self.external_networking()?.for_new_nexus()?; let external_ip = OmicronZoneExternalFloatingIp { - id: self.rng.external_ip_rng.next(), + id: self.rng.next_external_ip(), ip: external_ip, }; let nic = NetworkInterface { - id: self.rng.network_interface_rng.next(), + id: self.rng.next_network_interface(), kind: NetworkInterfaceKind::Service { id: nexus_id.into_untyped_uuid(), }, @@ -1148,7 +1412,12 @@ impl<'a> BlueprintBuilder<'a> { self.sled_add_zone(sled_id, zone)?; } - Ok(EnsureMultiple::Changed { added: num_nexus_to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: num_nexus_to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } pub fn sled_ensure_zone_multiple_oximeter( @@ -1173,7 +1442,7 @@ impl<'a> BlueprintBuilder<'a> { }; for _ in 0..num_oximeter_to_add { - let oximeter_id = self.rng.zone_rng.next(); + let oximeter_id = self.rng.next_zone(); let ip = self.sled_alloc_ip(sled_id)?; let port = omicron_common::address::OXIMETER_PORT; let address = SocketAddrV6::new(ip, port, 0, 0); @@ -1193,7 +1462,12 @@ impl<'a> BlueprintBuilder<'a> { self.sled_add_zone(sled_id, zone)?; } - Ok(EnsureMultiple::Changed { added: num_oximeter_to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: num_oximeter_to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } pub fn sled_ensure_zone_multiple_crucible_pantry( @@ -1218,7 +1492,7 @@ impl<'a> BlueprintBuilder<'a> { }; for _ in 0..num_pantry_to_add { - let pantry_id = self.rng.zone_rng.next(); + let pantry_id = self.rng.next_zone(); let ip = self.sled_alloc_ip(sled_id)?; let port = omicron_common::address::CRUCIBLE_PANTRY_PORT; let address = SocketAddrV6::new(ip, port, 0, 0); @@ -1237,7 +1511,12 @@ impl<'a> BlueprintBuilder<'a> { self.sled_add_zone(sled_id, zone)?; } - Ok(EnsureMultiple::Changed { added: num_pantry_to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: num_pantry_to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } pub fn cockroachdb_preserve_downgrade( @@ -1267,7 +1546,7 @@ impl<'a> BlueprintBuilder<'a> { } }; for _ in 0..num_crdb_to_add { - let zone_id = self.rng.zone_rng.next(); + let zone_id = self.rng.next_zone(); let underlay_ip = self.sled_alloc_ip(sled_id)?; let pool_name = self.sled_select_zpool(sled_id, ZoneKind::CockroachDb)?; @@ -1292,14 +1571,19 @@ impl<'a> BlueprintBuilder<'a> { self.sled_add_zone(sled_id, zone)?; } - Ok(EnsureMultiple::Changed { added: num_crdb_to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: num_crdb_to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } fn sled_add_zone_clickhouse( &mut self, sled_id: SledUuid, ) -> Result { - let id = self.rng.zone_rng.next(); + let id = self.rng.next_zone(); let underlay_address = self.sled_alloc_ip(sled_id)?; let address = SocketAddrV6::new(underlay_address, CLICKHOUSE_HTTP_PORT, 0, 0); @@ -1343,7 +1627,12 @@ impl<'a> BlueprintBuilder<'a> { for _ in 0..to_add { self.sled_add_zone_clickhouse(sled_id)?; } - Ok(EnsureMultiple::Changed { added: to_add, removed: 0 }) + Ok(EnsureMultiple::Changed { + added: to_add, + updated: 0, + expunged: 0, + removed: 0, + }) } pub fn sled_ensure_zone_multiple_clickhouse_server( @@ -1369,7 +1658,7 @@ impl<'a> BlueprintBuilder<'a> { } }; for _ in 0..num_clickhouse_servers_to_add { - let zone_id = self.rng.zone_rng.next(); + let zone_id = self.rng.next_zone(); let underlay_ip = self.sled_alloc_ip(sled_id)?; let pool_name = self.sled_select_zpool(sled_id, ZoneKind::ClickhouseServer)?; @@ -1396,6 +1685,8 @@ impl<'a> BlueprintBuilder<'a> { Ok(EnsureMultiple::Changed { added: num_clickhouse_servers_to_add, + updated: 0, + expunged: 0, removed: 0, }) } @@ -1424,7 +1715,7 @@ impl<'a> BlueprintBuilder<'a> { }; for _ in 0..num_clickhouse_keepers_to_add { - let zone_id = self.rng.zone_rng.next(); + let zone_id = self.rng.next_zone(); let underlay_ip = self.sled_alloc_ip(sled_id)?; let pool_name = self.sled_select_zpool(sled_id, ZoneKind::ClickhouseKeeper)?; @@ -1451,6 +1742,8 @@ impl<'a> BlueprintBuilder<'a> { Ok(EnsureMultiple::Changed { added: num_clickhouse_keepers_to_add, + updated: 0, + expunged: 0, removed: 0, }) } @@ -1533,7 +1826,7 @@ impl<'a> BlueprintBuilder<'a> { })?; // Add the new boundary NTP zone. - let new_zone_id = self.rng.zone_rng.next(); + let new_zone_id = self.rng.next_zone(); let ExternalSnatNetworkingChoice { snat_cfg, nic_ip, @@ -1541,11 +1834,11 @@ impl<'a> BlueprintBuilder<'a> { nic_mac, } = self.external_networking()?.for_new_boundary_ntp()?; let external_ip = OmicronZoneExternalSnatIp { - id: self.rng.external_ip_rng.next(), + id: self.rng.next_external_ip(), snat_cfg, }; let nic = NetworkInterface { - id: self.rng.network_interface_rng.next(), + id: self.rng.next_network_interface(), kind: NetworkInterfaceKind::Service { id: new_zone_id.into_untyped_uuid(), }, @@ -1583,7 +1876,12 @@ impl<'a> BlueprintBuilder<'a> { }, )?; - Ok(EnsureMultiple::Changed { added: 1, removed: 1 }) + Ok(EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 1, + }) } pub fn sled_expunge_zone( @@ -1749,49 +2047,6 @@ impl<'a> BlueprintBuilder<'a> { } } -#[derive(Debug)] -struct BlueprintBuilderRng { - // Have separate RNGs for the different kinds of UUIDs we might add, - // generated from the main RNG. This is so that e.g. adding a new network - // interface doesn't alter the blueprint or sled UUID. - // - // In the future, when we switch to typed UUIDs, each of these will be - // associated with a specific `TypedUuidKind`. - blueprint_rng: UuidRng, - zone_rng: TypedUuidRng, - network_interface_rng: UuidRng, - external_ip_rng: TypedUuidRng, -} - -impl BlueprintBuilderRng { - fn new() -> Self { - Self::new_from_parent(StdRng::from_entropy()) - } - - fn new_from_parent(mut parent: StdRng) -> Self { - let blueprint_rng = UuidRng::from_parent_rng(&mut parent, "blueprint"); - let zone_rng = TypedUuidRng::from_parent_rng(&mut parent, "zone"); - let network_interface_rng = - UuidRng::from_parent_rng(&mut parent, "network_interface"); - let external_ip_rng = - TypedUuidRng::from_parent_rng(&mut parent, "external_ip"); - - BlueprintBuilderRng { - blueprint_rng, - zone_rng, - network_interface_rng, - external_ip_rng, - } - } - - fn set_seed(&mut self, seed: H) { - // Important to add some more bytes here, so that builders with the - // same seed but different purposes don't end up with the same UUIDs. - const SEED_EXTRA: &str = "blueprint-builder"; - *self = Self::new_from_parent(typed_rng::from_seed(seed, SEED_EXTRA)); - } -} - /// Helper for working with sets of zones on each sled /// /// Tracking the set of zones is slightly non-trivial because we need to bump @@ -2018,6 +2273,335 @@ impl<'a> BlueprintDisksBuilder<'a> { } } +/// Helper for working with sets of datasets on each sled +struct BlueprintDatasetsBuilder<'a> { + changed_datasets: BTreeMap, + parent_datasets: &'a BTreeMap, +} + +impl<'a> BlueprintDatasetsBuilder<'a> { + pub fn new(parent_blueprint: &'a Blueprint) -> BlueprintDatasetsBuilder { + BlueprintDatasetsBuilder { + changed_datasets: BTreeMap::new(), + parent_datasets: &parent_blueprint.blueprint_datasets, + } + } + + pub fn change_sled_datasets( + &mut self, + sled_id: SledUuid, + ) -> &mut BlueprintDatasetsConfig { + self.changed_datasets.entry(sled_id).or_insert_with(|| { + if let Some(old_sled_datasets) = self.parent_datasets.get(&sled_id) + { + BlueprintDatasetsConfig { + generation: old_sled_datasets.generation.next(), + datasets: old_sled_datasets.datasets.clone(), + } + } else { + BlueprintDatasetsConfig { + generation: Generation::new(), + datasets: BTreeMap::new(), + } + } + }) + } + + /// Iterates over the list of Omicron datasets currently configured for this + /// sled in the blueprint that's being built + pub fn current_sled_datasets( + &self, + sled_id: SledUuid, + ) -> Box + '_> { + if let Some(sled_datasets) = self + .changed_datasets + .get(&sled_id) + .or_else(|| self.parent_datasets.get(&sled_id)) + { + Box::new(sled_datasets.datasets.values()) + } else { + Box::new(std::iter::empty()) + } + } + + /// Produces an owned map of datasets for the requested sleds + pub fn into_datasets_map( + mut self, + sled_ids: impl Iterator, + ) -> BTreeMap { + sled_ids + .map(|sled_id| { + // Start with self.changed_datasets, which contains entries for any + // sled whose datasets config is changing in this blueprint. + let datasets = self + .changed_datasets + .remove(&sled_id) + // If it's not there, use the config from the parent + // blueprint. + .or_else(|| self.parent_datasets.get(&sled_id).cloned()) + // If it's not there either, then this must be a new sled + // and we haven't added any datasets to it yet. Use the + // standard initial config. + .unwrap_or_else(|| BlueprintDatasetsConfig { + generation: Generation::new(), + datasets: BTreeMap::new(), + }); + + (sled_id, datasets) + }) + .collect() + } +} + +/// Helper for working with sets of datasets on a single sled +#[derive(Debug)] +struct BlueprintSledDatasetsBuilder<'a> { + log: Logger, + blueprint_datasets: + BTreeMap>, + database_datasets: + BTreeMap>, + + // Datasets which are unchanged from the prior blueprint + unchanged_datasets: + BTreeMap>, + // Datasets which are new in this blueprint + new_datasets: + BTreeMap>, + // Datasets which existed in the old blueprint, but which are + // changing in this one + updated_datasets: + BTreeMap>, +} + +impl<'a> BlueprintSledDatasetsBuilder<'a> { + pub fn new( + log: Logger, + sled_id: SledUuid, + datasets: &'a BlueprintDatasetsBuilder<'_>, + resources: &'a SledResources, + ) -> Self { + // Gather all datasets known to the blueprint + let mut blueprint_datasets: BTreeMap< + ZpoolUuid, + BTreeMap, + > = BTreeMap::new(); + for dataset in datasets.current_sled_datasets(sled_id) { + blueprint_datasets + .entry(dataset.pool.id()) + .or_default() + .insert(dataset.kind.clone(), dataset); + } + + // Gather all datasets known to the database + let mut database_datasets = BTreeMap::new(); + for (zpool, datasets) in resources.all_datasets(ZpoolFilter::InService) + { + let datasets_by_kind = datasets + .into_iter() + .map(|dataset| (dataset.name.dataset().clone(), dataset)) + .collect(); + + database_datasets.insert(*zpool, datasets_by_kind); + } + + Self { + log, + blueprint_datasets, + database_datasets, + unchanged_datasets: BTreeMap::new(), + new_datasets: BTreeMap::new(), + updated_datasets: BTreeMap::new(), + } + } + + /// Attempts to add a dataset to the builder. + /// + /// - If the dataset exists in the blueprint already, use it. + /// - Otherwise, if the dataset exists in the database, re-use the UUID, but + /// add it to the blueprint. + /// - Otherwse, create a new dataset in the blueprint, which will propagate + /// to the database during execution. + pub fn ensure( + &mut self, + dataset: DatasetName, + address: Option, + quota: Option, + reservation: Option, + compression: CompressionAlgorithm, + ) { + let zpool = dataset.pool(); + let zpool_id = zpool.id(); + let kind = dataset.dataset(); + + let make_config = |id: DatasetUuid| BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id, + pool: zpool.clone(), + kind: kind.clone(), + address, + quota, + reservation, + compression, + }; + + // This dataset already exists in the blueprint + if let Some(old_config) = self.get_from_bp(zpool_id, kind) { + let new_config = make_config(old_config.id); + + // If it needs updating, add it + let target = if *old_config != new_config { + &mut self.updated_datasets + } else { + &mut self.unchanged_datasets + }; + target + .entry(zpool_id) + .or_default() + .insert(new_config.kind.clone(), new_config); + return; + } + + // If the dataset exists in the datastore, re-use the UUID. + // + // TODO(https://github.com/oxidecomputer/omicron/issues/6645): We + // could avoid reading from the datastore if we were confident all + // provisioned datasets existed in the parent blueprint. + let id = if let Some(old_config) = self.get_from_db(zpool_id, kind) { + old_config.id + } else { + DatasetUuid::new_v4() + }; + + let new_config = make_config(id); + self.new_datasets + .entry(zpool_id) + .or_default() + .insert(new_config.kind.clone(), new_config); + } + + /// Returns all datasets in the old blueprint that are not planned to be + /// part of the new blueprint. + pub fn get_expungeable_datasets(&self) -> BTreeSet { + let dataset_exists_in = + |group: &BTreeMap< + ZpoolUuid, + BTreeMap, + >, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid| { + let Some(datasets) = group.get(&zpool_id) else { + return false; + }; + + datasets.values().any(|config| config.id == dataset_id) + }; + + let mut expunges = BTreeSet::new(); + + for (zpool_id, datasets) in &self.blueprint_datasets { + for dataset_config in datasets.values() { + match dataset_config.disposition { + // Already expunged; ignore + BlueprintDatasetDisposition::Expunged => continue, + // Potentially expungeable + BlueprintDatasetDisposition::InService => (), + }; + + let dataset_id = dataset_config.id; + if !dataset_exists_in(&self.new_datasets, *zpool_id, dataset_id) + && !dataset_exists_in( + &self.updated_datasets, + *zpool_id, + dataset_id, + ) + && !dataset_exists_in( + &self.unchanged_datasets, + *zpool_id, + dataset_id, + ) + { + info!(self.log, "dataset expungeable (not needed in blueprint)"; "id" => ?dataset_id); + expunges.insert(dataset_id); + } + } + } + + expunges + } + + /// TODO: + /// This function SHOULD do the following: + /// + /// Returns all datasets that have been expunged in a prior blueprint, and + /// which have also been removed from the database and from inventory. + /// This is our sign that the work of expungement has completed. + /// + /// TODO: In reality, however, this function actually implements the + /// following: + /// + /// - It returns an empty BTreeSet, effectively saying "no datasets are + /// removable from the blueprint". + pub fn get_removable_datasets(&self) -> BTreeSet { + let dataset_exists_in = + |group: &BTreeMap< + ZpoolUuid, + BTreeMap, + >, + zpool_id: ZpoolUuid, + dataset_id: DatasetUuid| { + let Some(datasets) = group.get(&zpool_id) else { + return false; + }; + + datasets.values().any(|config| config.id == dataset_id) + }; + + let removals = BTreeSet::new(); + for (zpool_id, datasets) in &self.blueprint_datasets { + for (_kind, config) in datasets { + if config.disposition == BlueprintDatasetDisposition::Expunged + && !dataset_exists_in( + &self.database_datasets, + *zpool_id, + config.id, + ) + { + info!(self.log, "dataset removable (expunged, not in database)"; "id" => ?config.id); + + // TODO(https://github.com/oxidecomputer/omicron/issues/6646): + // We could call `removals.insert(config.id)` here, but + // instead, opt to just log that the dataset is removable + // and keep it in the blueprint. + } + } + } + removals + } + + fn get_from_bp( + &self, + zpool: ZpoolUuid, + kind: &DatasetKind, + ) -> Option<&'a BlueprintDatasetConfig> { + self.blueprint_datasets + .get(&zpool) + .and_then(|datasets| datasets.get(kind)) + .copied() + } + + fn get_from_db( + &self, + zpool: ZpoolUuid, + kind: &DatasetKind, + ) -> Option<&'a DatasetConfig> { + self.database_datasets + .get(&zpool) + .and_then(|datasets| datasets.get(kind)) + .copied() + } +} + #[cfg(test)] pub mod test { use super::*; @@ -2036,6 +2620,35 @@ pub mod test { use std::collections::BTreeSet; use std::mem; + pub const DEFAULT_N_SLEDS: usize = 3; + + fn datasets_for_sled( + blueprint: &Blueprint, + sled_id: SledUuid, + ) -> &BTreeMap { + &blueprint + .blueprint_datasets + .get(&sled_id) + .unwrap_or_else(|| { + panic!("Cannot find datasets on missing sled: {sled_id}") + }) + .datasets + } + + fn find_dataset<'a>( + datasets: &'a BTreeMap, + zpool: &ZpoolName, + kind: DatasetKind, + ) -> &'a BlueprintDatasetConfig { + datasets.values().find(|dataset| { + &dataset.pool == zpool && + dataset.kind == kind + }).unwrap_or_else(|| { + let kinds = datasets.values().map(|d| (&d.id, &d.pool, &d.kind)).collect::>(); + panic!("Cannot find dataset of type {kind}\nFound the following: {kinds:#?}") + }) + } + /// Checks various conditions that should be true for all blueprints #[track_caller] pub fn verify_blueprint(blueprint: &Blueprint) { @@ -2088,6 +2701,66 @@ pub mod test { } } } + + // All commissioned disks should have debug and zone root datasets. + for (sled_id, disk_config) in &blueprint.blueprint_disks { + for disk in &disk_config.disks { + let zpool = ZpoolName::new_external(disk.pool_id); + let datasets = datasets_for_sled(&blueprint, *sled_id); + + let dataset = + find_dataset(&datasets, &zpool, DatasetKind::Debug); + assert_eq!( + dataset.disposition, + BlueprintDatasetDisposition::InService + ); + let dataset = find_dataset( + &datasets, + &zpool, + DatasetKind::TransientZoneRoot, + ); + assert_eq!( + dataset.disposition, + BlueprintDatasetDisposition::InService + ); + } + } + // All zones should have dataset records. + for (sled_id, zone_config) in + blueprint.all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + { + match blueprint.sled_state.get(&sled_id) { + // Decommissioned sleds don't keep dataset state around. + // + // Normally we wouldn't observe zones from decommissioned sleds + // anyway, but that's the responsibility of the Planner, not the + // BlueprintBuilder. + None | Some(SledState::Decommissioned) => continue, + Some(SledState::Active) => (), + } + let datasets = datasets_for_sled(&blueprint, sled_id); + + let zpool = zone_config.filesystem_pool.as_ref().unwrap(); + let kind = + DatasetKind::TransientZone { name: zone_name(&zone_config) }; + let dataset = find_dataset(&datasets, &zpool, kind); + assert_eq!( + dataset.disposition, + BlueprintDatasetDisposition::InService + ); + + if let Some(durable_dataset) = + zone_config.zone_type.durable_dataset() + { + let zpool = &durable_dataset.dataset.pool_name; + let dataset = + find_dataset(&datasets, &zpool, durable_dataset.kind); + assert_eq!( + dataset.disposition, + BlueprintDatasetDisposition::InService + ); + } + } } #[track_caller] @@ -2217,6 +2890,7 @@ pub mod test { for pool_id in new_sled_resources.zpools.keys() { builder.sled_ensure_zone_crucible(new_sled_id, *pool_id).unwrap(); } + builder.sled_ensure_datasets(new_sled_id, new_sled_resources).unwrap(); let blueprint3 = builder.build(); verify_blueprint(&blueprint3); @@ -2444,6 +3118,8 @@ pub mod test { .unwrap(), EnsureMultiple::Changed { added: usize::from(SledBuilder::DEFAULT_NPOOLS), + updated: 0, + expunged: 0, removed: 0 }, ); @@ -2490,6 +3166,163 @@ pub mod test { logctx.cleanup_successful(); } + #[test] + fn test_datasets_for_zpools_and_zones() { + static TEST_NAME: &str = "test_datasets_for_zpools_and_zones"; + let logctx = test_setup_log(TEST_NAME); + let (collection, input, blueprint) = example(&logctx.log, TEST_NAME); + + // Creating the "example" blueprint should already invoke + // `sled_ensure_datasets`. + // + // Verify that it has created the datasets we expect to exist. + verify_blueprint(&blueprint); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &input, + &collection, + "test", + ) + .expect("failed to create builder"); + + // Before we make any modifications, there should be no work to do. + // + // If we haven't changed inputs, the output should be the same! + for (sled_id, resources) in + input.all_sled_resources(SledFilter::Commissioned) + { + let r = builder.sled_ensure_datasets(sled_id, resources).unwrap(); + assert_eq!(r, EnsureMultiple::NotNeeded); + } + + // Expunge a zone from the blueprint, observe that the dataset is + // removed. + let sled_id = input + .all_sled_ids(SledFilter::Commissioned) + .next() + .expect("at least one sled present"); + let sled_details = + input.sled_lookup(SledFilter::Commissioned, sled_id).unwrap(); + let crucible_zone_id = builder + .zones + .current_sled_zones(sled_id, BlueprintZoneFilter::ShouldBeRunning) + .find_map(|(zone_config, _)| { + if zone_config.zone_type.is_crucible() { + return Some(zone_config.id); + } + None + }) + .expect("at least one crucible must be present"); + let change = builder.zones.change_sled_zones(sled_id); + println!("Expunging crucible zone: {crucible_zone_id}"); + change.expunge_zones(BTreeSet::from([crucible_zone_id])).unwrap(); + + // In the case of Crucible, we have a durable dataset and a transient + // zone filesystem, so we expect two datasets to be expunged. + let r = builder + .sled_ensure_datasets(sled_id, &sled_details.resources) + .unwrap(); + assert_eq!( + r, + EnsureMultiple::Changed { + added: 0, + updated: 0, + expunged: 2, + removed: 0 + } + ); + // Once the datasets are expunged, no further changes will be proposed. + let r = builder + .sled_ensure_datasets(sled_id, &sled_details.resources) + .unwrap(); + assert_eq!(r, EnsureMultiple::NotNeeded); + + let blueprint = builder.build(); + verify_blueprint(&blueprint); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &input, + &collection, + "test", + ) + .expect("failed to create builder"); + + // While the datasets still exist in the input (effectively, the db) we + // cannot remove them. + let r = builder + .sled_ensure_datasets(sled_id, &sled_details.resources) + .unwrap(); + assert_eq!(r, EnsureMultiple::NotNeeded); + + let blueprint = builder.build(); + verify_blueprint(&blueprint); + + // Find the datasets we've expunged in the blueprint + let expunged_datasets = blueprint + .blueprint_datasets + .get(&sled_id) + .unwrap() + .datasets + .values() + .filter_map(|dataset_config| { + if dataset_config.disposition + == BlueprintDatasetDisposition::Expunged + { + Some(dataset_config.id) + } else { + None + } + }) + .collect::>(); + // We saw two datasets being expunged earlier when we called + // `sled_ensure_datasets` -- validate that this is true when inspecting + // the blueprint too. + assert_eq!(expunged_datasets.len(), 2); + + // Remove these two datasets from the input. + let mut input_builder = input.into_builder(); + let zpools = &mut input_builder + .sleds_mut() + .get_mut(&sled_id) + .unwrap() + .resources + .zpools; + for (_, (_, datasets)) in zpools { + datasets.retain(|dataset| !expunged_datasets.contains(&dataset.id)); + } + let input = input_builder.build(); + + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &blueprint, + &input, + &collection, + "test", + ) + .expect("failed to create builder"); + + // Now, we should see the datasets "removed" from the blueprint, since + // we no longer need to keep around records of their expungement. + let sled_details = + input.sled_lookup(SledFilter::Commissioned, sled_id).unwrap(); + let r = builder + .sled_ensure_datasets(sled_id, &sled_details.resources) + .unwrap(); + + // TODO(https://github.com/oxidecomputer/omicron/issues/6646): + // Because of the workaround for #6646, we don't actually remove + // datasets yet. + // + // In the future, however, we will. + assert_eq!(r, EnsureMultiple::NotNeeded); + + logctx.cleanup_successful(); + } + #[test] fn test_add_nexus_with_no_existing_nexus_zones() { static TEST_NAME: &str = @@ -2620,7 +3453,15 @@ pub mod test { .sled_ensure_zone_multiple_nexus(sled_id, 1) .expect("failed to ensure nexus zone"); - assert_eq!(added, EnsureMultiple::Changed { added: 1, removed: 0 }); + assert_eq!( + added, + EnsureMultiple::Changed { + added: 1, + updated: 0, + expunged: 0, + removed: 0 + } + ); } { @@ -2639,7 +3480,15 @@ pub mod test { .sled_ensure_zone_multiple_nexus(sled_id, 3) .expect("failed to ensure nexus zone"); - assert_eq!(added, EnsureMultiple::Changed { added: 3, removed: 0 }); + assert_eq!( + added, + EnsureMultiple::Changed { + added: 3, + updated: 0, + expunged: 0, + removed: 0 + } + ); } { @@ -2907,8 +3756,14 @@ pub mod test { .expect("ensured multiple CRDB zones"); assert_eq!( ensure_result, - EnsureMultiple::Changed { added: num_sled_zpools, removed: 0 } + EnsureMultiple::Changed { + added: num_sled_zpools, + updated: 0, + expunged: 0, + removed: 0 + } ); + builder.sled_ensure_datasets(target_sled_id, sled_resources).unwrap(); let blueprint = builder.build(); verify_blueprint(&blueprint); diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs b/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs index 3b95a60ad8..2d9194ee52 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs @@ -271,16 +271,20 @@ mod tests { ), zpools: BTreeMap::from([( ZpoolUuid::new_v4(), - SledDisk { - disk_identity: DiskIdentity { - vendor: String::from("fake-vendor"), - serial: String::from("fake-serial"), - model: String::from("fake-model"), + ( + SledDisk { + disk_identity: DiskIdentity { + vendor: String::from("fake-vendor"), + serial: String::from("fake-serial"), + model: String::from("fake-model"), + }, + disk_id: PhysicalDiskUuid::new_v4(), + policy: PhysicalDiskPolicy::InService, + state: PhysicalDiskState::Active, }, - disk_id: PhysicalDiskUuid::new_v4(), - policy: PhysicalDiskPolicy::InService, - state: PhysicalDiskState::Active, - }, + // Datasets: Leave empty + vec![], + ), )]), }, }, @@ -421,6 +425,13 @@ mod tests { } ); + // Ensure all datasets are created for the zones we've provisioned + for (sled_id, resources) in + input2.all_sled_resources(SledFilter::Commissioned) + { + builder.sled_ensure_datasets(sled_id, resources).unwrap(); + } + // Now build the blueprint and ensure that all the changes we described // above are present. let blueprint = builder.build(); diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index 55919b1c79..a0761a9b6c 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -345,6 +345,7 @@ impl ExampleSystemBuilder { .unwrap(); } } + builder.sled_ensure_datasets(sled_id, &sled_resources).unwrap(); } let blueprint = builder.build(); @@ -388,6 +389,27 @@ impl ExampleSystemBuilder { .unwrap(); } + // Ensure that our "input" contains the datasets we would have + // provisioned. + // + // This mimics them existing within the database. + let input_sleds = input_builder.sleds_mut(); + for (sled_id, bp_datasets_config) in &blueprint.blueprint_datasets { + let sled = input_sleds.get_mut(sled_id).unwrap(); + for (_, bp_dataset) in &bp_datasets_config.datasets { + let (_, datasets) = sled + .resources + .zpools + .get_mut(&bp_dataset.pool.id()) + .unwrap(); + let bp_config: omicron_common::disk::DatasetConfig = + bp_dataset.clone().try_into().unwrap(); + if !datasets.contains(&bp_config) { + datasets.push(bp_config); + } + } + } + let mut builder = system.to_collection_builder().expect("failed to build collection"); builder.set_rng_seed((&self.test_name, "ExampleSystem collection")); diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 7a20f57017..95a621186a 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -40,6 +40,7 @@ use self::omicron_zone_placement::OmicronZonePlacement; use self::omicron_zone_placement::OmicronZonePlacementSledState; mod omicron_zone_placement; +pub(crate) mod rng; pub struct Planner<'a> { log: Logger, @@ -231,8 +232,12 @@ impl<'a> Planner<'a> { { // First, we need to ensure that sleds are using their expected // disks. This is necessary before we can allocate any zones. - if let EnsureMultiple::Changed { added, removed } = - self.blueprint.sled_ensure_disks(sled_id, &sled_resources)? + if let EnsureMultiple::Changed { + added, + updated, + expunged: _, + removed, + } = self.blueprint.sled_ensure_disks(sled_id, &sled_resources)? { info!( &self.log, @@ -242,6 +247,7 @@ impl<'a> Planner<'a> { self.blueprint.record_operation(Operation::UpdateDisks { sled_id, added, + updated, removed, }); @@ -344,7 +350,46 @@ impl<'a> Planner<'a> { } } - self.do_plan_add_discretionary_zones(&sleds_waiting_for_ntp_zone) + self.do_plan_add_discretionary_zones(&sleds_waiting_for_ntp_zone)?; + + // Now that we've added all the disks and zones we plan on adding, + // ensure that all sleds have the datasets they need to have. + self.do_plan_datasets()?; + + Ok(()) + } + + fn do_plan_datasets(&mut self) -> Result<(), Error> { + for (sled_id, sled_resources) in + self.input.all_sled_resources(SledFilter::InService) + { + if let EnsureMultiple::Changed { + added, + updated, + expunged, + removed, + } = + self.blueprint.sled_ensure_datasets(sled_id, &sled_resources)? + { + info!( + &self.log, + "altered datasets"; + "sled_id" => %sled_id, + "added" => added, + "updated" => updated, + "expunged" => expunged, + "removed" => removed, + ); + self.blueprint.record_operation(Operation::UpdateDatasets { + sled_id, + added, + updated, + expunged, + removed, + }); + } + } + Ok(()) } fn do_plan_add_discretionary_zones( @@ -598,12 +643,19 @@ impl<'a> Planner<'a> { } }; match result { - EnsureMultiple::Changed { added, removed } => { + EnsureMultiple::Changed { + added, + updated, + expunged, + removed, + } => { info!( self.log, "modified zones on sled"; "sled_id" => %sled_id, "kind" => ?kind, "added" => added, + "updated" => updated, + "expunged" => expunged, "removed" => removed, ); new_zones_added += added; @@ -1445,7 +1497,12 @@ mod test { builder .sled_ensure_zone_multiple_external_dns(sled_id, 3) .expect("can't add external DNS zones"), - EnsureMultiple::Changed { added: 0, removed: 0 }, + EnsureMultiple::Changed { + added: 0, + updated: 0, + removed: 0, + expunged: 0 + }, ); // Build a builder for a modfied blueprint that will include @@ -1484,13 +1541,23 @@ mod test { blueprint_builder .sled_ensure_zone_multiple_external_dns(sled_1, 2) .expect("can't add external DNS zones to blueprint"), - EnsureMultiple::Changed { added: 2, removed: 0 } + EnsureMultiple::Changed { + added: 2, + updated: 0, + removed: 0, + expunged: 0 + } )); assert!(matches!( blueprint_builder .sled_ensure_zone_multiple_external_dns(sled_2, 1) .expect("can't add external DNS zones to blueprint"), - EnsureMultiple::Changed { added: 1, removed: 0 } + EnsureMultiple::Changed { + added: 1, + updated: 0, + removed: 0, + expunged: 0 + } )); let blueprint1a = blueprint_builder.build(); @@ -1641,13 +1708,13 @@ mod test { for _ in 0..NEW_IN_SERVICE_DISKS { sled_details.resources.zpools.insert( ZpoolUuid::from(zpool_rng.next()), - new_sled_disk(PhysicalDiskPolicy::InService), + (new_sled_disk(PhysicalDiskPolicy::InService), vec![]), ); } for _ in 0..NEW_EXPUNGED_DISKS { sled_details.resources.zpools.insert( ZpoolUuid::from(zpool_rng.next()), - new_sled_disk(PhysicalDiskPolicy::Expunged), + (new_sled_disk(PhysicalDiskPolicy::Expunged), vec![]), ); } @@ -1726,7 +1793,7 @@ mod test { } } let (_, sled_details) = builder.sleds_mut().iter_mut().next().unwrap(); - let (_, disk) = sled_details + let (_, (disk, _datasets)) = sled_details .resources .zpools .iter_mut() @@ -1851,7 +1918,7 @@ mod test { // For that pool, find the physical disk behind it, and mark it // expunged. let (_, sled_details) = builder.sleds_mut().iter_mut().next().unwrap(); - let disk = sled_details + let (disk, _datasets) = sled_details .resources .zpools .get_mut(&pool_to_expunge.id()) diff --git a/nexus/reconfigurator/planning/src/planner/rng.rs b/nexus/reconfigurator/planning/src/planner/rng.rs new file mode 100644 index 0000000000..28ea9b336c --- /dev/null +++ b/nexus/reconfigurator/planning/src/planner/rng.rs @@ -0,0 +1,71 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! RNG for blueprint planning to allow reproducibility (particularly for +//! tests). + +use omicron_uuid_kinds::ExternalIpKind; +use omicron_uuid_kinds::ExternalIpUuid; +use omicron_uuid_kinds::OmicronZoneKind; +use omicron_uuid_kinds::OmicronZoneUuid; +use rand::rngs::StdRng; +use rand::SeedableRng as _; +use std::hash::Hash; +use typed_rng::TypedUuidRng; +use typed_rng::UuidRng; +use uuid::Uuid; + +#[derive(Debug)] +pub(crate) struct PlannerRng { + // Have separate RNGs for the different kinds of UUIDs we might add, + // generated from the main RNG. This is so that e.g. adding a new network + // interface doesn't alter the blueprint or sled UUID. + // + // In the future, when we switch to typed UUIDs, each of these will be + // associated with a specific `TypedUuidKind`. + blueprint_rng: UuidRng, + zone_rng: TypedUuidRng, + network_interface_rng: UuidRng, + external_ip_rng: TypedUuidRng, +} + +impl PlannerRng { + pub fn new() -> Self { + Self::new_from_parent(StdRng::from_entropy()) + } + + pub fn new_from_parent(mut parent: StdRng) -> Self { + let blueprint_rng = UuidRng::from_parent_rng(&mut parent, "blueprint"); + let zone_rng = TypedUuidRng::from_parent_rng(&mut parent, "zone"); + let network_interface_rng = + UuidRng::from_parent_rng(&mut parent, "network_interface"); + let external_ip_rng = + TypedUuidRng::from_parent_rng(&mut parent, "external_ip"); + + Self { blueprint_rng, zone_rng, network_interface_rng, external_ip_rng } + } + + pub fn set_seed(&mut self, seed: H) { + // Important to add some more bytes here, so that builders with the + // same seed but different purposes don't end up with the same UUIDs. + const SEED_EXTRA: &str = "blueprint-builder"; + *self = Self::new_from_parent(typed_rng::from_seed(seed, SEED_EXTRA)); + } + + pub fn next_blueprint(&mut self) -> Uuid { + self.blueprint_rng.next() + } + + pub fn next_zone(&mut self) -> OmicronZoneUuid { + self.zone_rng.next() + } + + pub fn next_network_interface(&mut self) -> Uuid { + self.network_interface_rng.next() + } + + pub fn next_external_ip(&mut self) -> ExternalIpUuid { + self.external_ip_rng.next() + } +} diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 10f879e7d6..9ffd99bd5f 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -585,7 +585,8 @@ impl Sled { policy: PhysicalDiskPolicy::InService, state: PhysicalDiskState::Active, }; - (zpool, disk) + let datasets = vec![]; + (zpool, (disk, datasets)) }) .collect(); let inventory_sp = match hardware { @@ -648,8 +649,8 @@ impl Sled { disks: zpools .values() .enumerate() - .map(|(i, d)| InventoryDisk { - identity: d.disk_identity.clone(), + .map(|(i, (disk, _datasets))| InventoryDisk { + identity: disk.disk_identity.clone(), variant: DiskVariant::U2, slot: i64::try_from(i).unwrap(), active_firmware_slot: 1, diff --git a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt index b0b82a31f8..072e26df49 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt @@ -114,7 +114,7 @@ WARNING: Zones exist without physical disks! METADATA: created by::::::::::: test_blueprint2 created at::::::::::: 1970-01-01T00:00:00.000Z - comment:::::::::::::: sled a1b477db-b629-48eb-911d-1ccdafca75b9: expunged 15 zones because: sled policy is expunged + comment:::::::::::::: sled a1b477db-b629-48eb-911d-1ccdafca75b9: expunged 15 zones because: sled policy is expunged, sled d67ce8f0-a691-4010-b414-420d82e80527: added 2 datasets, updated: 0, expunged 0, removed 0 datasets, sled fefcf4cf-f7e7-46b3-b629-058526ce440e: added 4 datasets, updated: 0, expunged 0, removed 0 datasets internal DNS version: 1 external DNS version: 1 diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt index 04c119ea9f..5d777cd9d9 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt @@ -175,7 +175,7 @@ WARNING: Zones exist without physical disks! METADATA: created by::::::::::: test_blueprint2 created at::::::::::: 1970-01-01T00:00:00.000Z - comment:::::::::::::: sled 48d95fef-bc9f-4f50-9a53-1e075836291d: expunged 14 zones because: sled policy is expunged + comment:::::::::::::: sled 48d95fef-bc9f-4f50-9a53-1e075836291d: expunged 14 zones because: sled policy is expunged, sled 75bc286f-2b4b-482c-9431-59272af529da: added 3 datasets, updated: 0, expunged 0, removed 0 datasets, sled affab35f-600a-4109-8ea0-34a067a4e0bc: added 3 datasets, updated: 0, expunged 0, removed 0 datasets internal DNS version: 1 external DNS version: 1 diff --git a/nexus/reconfigurator/preparation/src/lib.rs b/nexus/reconfigurator/preparation/src/lib.rs index 24f32e9187..39367824b9 100644 --- a/nexus/reconfigurator/preparation/src/lib.rs +++ b/nexus/reconfigurator/preparation/src/lib.rs @@ -38,6 +38,7 @@ use omicron_common::address::SLED_PREFIX; use omicron_common::api::external::Error; use omicron_common::api::external::InternalContext; use omicron_common::api::external::LookupType; +use omicron_common::disk::DatasetConfig; use omicron_common::disk::DiskIdentity; use omicron_common::policy::BOUNDARY_NTP_REDUNDANCY; use omicron_common::policy::COCKROACHDB_REDUNDANCY; @@ -63,6 +64,7 @@ pub struct PlanningInputFromDb<'a> { pub sled_rows: &'a [nexus_db_model::Sled], pub zpool_rows: &'a [(nexus_db_model::Zpool, nexus_db_model::PhysicalDisk)], + pub dataset_rows: &'a [nexus_db_model::Dataset], pub ip_pool_range_rows: &'a [nexus_db_model::IpPoolRange], pub external_ip_rows: &'a [nexus_db_model::ExternalIp], pub service_nic_rows: &'a [nexus_db_model::ServiceNetworkInterface], @@ -107,6 +109,10 @@ impl PlanningInputFromDb<'_> { .zpool_list_all_external_batched(opctx) .await .internal_context("fetching all external zpool rows")?; + let dataset_rows = datastore + .dataset_list_all_batched(opctx, None) + .await + .internal_context("fetching all datasets")?; let ip_pool_range_rows = { let (authz_service_ip_pool, _) = datastore .ip_pools_service_lookup(opctx) @@ -148,6 +154,7 @@ impl PlanningInputFromDb<'_> { let planning_input = PlanningInputFromDb { sled_rows: &sled_rows, zpool_rows: &zpool_rows, + dataset_rows: &dataset_rows, ip_pool_range_rows: &ip_pool_range_rows, target_boundary_ntp_zone_count: BOUNDARY_NTP_REDUNDANCY, target_nexus_zone_count: NEXUS_REDUNDANCY, @@ -195,6 +202,27 @@ impl PlanningInputFromDb<'_> { ); let mut zpools_by_sled_id = { + // Gather all the datasets first, by Zpool ID + let mut datasets: Vec<_> = self + .dataset_rows + .iter() + .map(|dataset| { + ( + ZpoolUuid::from_untyped_uuid(dataset.pool_id), + dataset.clone(), + ) + }) + .collect(); + datasets.sort_unstable_by_key(|(zpool_id, _)| *zpool_id); + let mut datasets_by_zpool: BTreeMap<_, Vec<_>> = BTreeMap::new(); + for (zpool_id, dataset) in datasets { + datasets_by_zpool + .entry(zpool_id) + .or_default() + .push(DatasetConfig::try_from(dataset)?); + } + + // Iterate over all Zpools, identifying their disks and datasets let mut zpools = BTreeMap::new(); for (zpool, disk) in self.zpool_rows { let sled_zpool_names = @@ -211,7 +239,10 @@ impl PlanningInputFromDb<'_> { state: disk.disk_state.into(), }; - sled_zpool_names.insert(zpool_id, disk); + let datasets = datasets_by_zpool + .remove(&zpool_id) + .unwrap_or_else(|| vec![]); + sled_zpool_names.insert(zpool_id, (disk, datasets)); } zpools }; diff --git a/nexus/src/app/background/tasks/blueprint_execution.rs b/nexus/src/app/background/tasks/blueprint_execution.rs index fa0283c942..a9d47af117 100644 --- a/nexus/src/app/background/tasks/blueprint_execution.rs +++ b/nexus/src/app/background/tasks/blueprint_execution.rs @@ -185,9 +185,10 @@ mod test { }; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::{ - blueprint_zone_type, Blueprint, BlueprintPhysicalDisksConfig, - BlueprintTarget, BlueprintZoneConfig, BlueprintZoneDisposition, - BlueprintZoneType, BlueprintZonesConfig, CockroachDbPreserveDowngrade, + blueprint_zone_type, Blueprint, BlueprintDatasetsConfig, + BlueprintPhysicalDisksConfig, BlueprintTarget, BlueprintZoneConfig, + BlueprintZoneDisposition, BlueprintZoneType, BlueprintZonesConfig, + CockroachDbPreserveDowngrade, }; use nexus_types::external_api::views::SledState; use omicron_common::api::external::Generation; @@ -213,6 +214,7 @@ mod test { opctx: &OpContext, blueprint_zones: BTreeMap, blueprint_disks: BTreeMap, + blueprint_datasets: BTreeMap, dns_version: Generation, ) -> (BlueprintTarget, Blueprint) { let id = Uuid::new_v4(); @@ -240,6 +242,7 @@ mod test { id, blueprint_zones, blueprint_disks, + blueprint_datasets, sled_state, cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, @@ -367,6 +370,7 @@ mod test { &opctx, BTreeMap::new(), BTreeMap::new(), + BTreeMap::new(), generation, ) .await, @@ -433,6 +437,7 @@ mod test { (sled_id2, make_zones(BlueprintZoneDisposition::Quiesced)), ]), BTreeMap::new(), + BTreeMap::new(), generation, ) .await; diff --git a/nexus/src/app/background/tasks/blueprint_load.rs b/nexus/src/app/background/tasks/blueprint_load.rs index 8b5c02dd80..9ce1e89df4 100644 --- a/nexus/src/app/background/tasks/blueprint_load.rs +++ b/nexus/src/app/background/tasks/blueprint_load.rs @@ -218,6 +218,7 @@ mod test { id, blueprint_zones: BTreeMap::new(), blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state: BTreeMap::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, diff --git a/nexus/src/app/background/tasks/crdb_node_id_collector.rs b/nexus/src/app/background/tasks/crdb_node_id_collector.rs index f5ec6406fa..1cac4e8c5a 100644 --- a/nexus/src/app/background/tasks/crdb_node_id_collector.rs +++ b/nexus/src/app/background/tasks/crdb_node_id_collector.rs @@ -236,10 +236,9 @@ mod tests { use httptest::responders::json_encoded; use httptest::responders::status_code; use httptest::Expectation; - use nexus_db_queries::db::datastore::pub_test_utils::datastore_test; + use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; - use nexus_test_utils::db::test_setup_database; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; use omicron_common::zpool_name::ZpoolName; @@ -249,7 +248,6 @@ mod tests { use std::collections::BTreeMap; use std::iter; use std::net::SocketAddr; - use uuid::Uuid; // The `CockroachAdminFromBlueprintViaFixedPort` type above is the standard // way to map from a blueprint to an iterator of cockroach-admin addresses. @@ -345,9 +343,8 @@ mod tests { #[tokio::test] async fn test_activate_fails_if_no_blueprint() { let logctx = dev::test_setup_log("test_activate_fails_if_no_blueprint"); - let mut db = test_setup_database(&logctx.log).await; - let (opctx, datastore) = - datastore_test(&logctx.log, &db, Uuid::new_v4()).await; + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); let (_tx_blueprint, rx_blueprint) = watch::channel(None); let mut collector = @@ -356,8 +353,7 @@ mod tests { assert_eq!(result, json!({"error": "no blueprint"})); - datastore.terminate().await; - db.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } @@ -377,9 +373,8 @@ mod tests { async fn test_activate_with_no_unknown_node_ids() { let logctx = dev::test_setup_log("test_activate_with_no_unknown_node_ids"); - let mut db = test_setup_database(&logctx.log).await; - let (opctx, datastore) = - datastore_test(&logctx.log, &db, Uuid::new_v4()).await; + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); let blueprint = BlueprintBuilder::build_empty_with_sleds( iter::once(SledUuid::new_v4()), @@ -433,8 +428,7 @@ mod tests { .await; assert_eq!(result, json!({"nsuccess": crdb_zones.len()})); - datastore.terminate().await; - db.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } @@ -442,9 +436,8 @@ mod tests { async fn test_activate_with_unknown_node_ids() { // Test setup. let logctx = dev::test_setup_log("test_activate_with_unknown_node_ids"); - let mut db = test_setup_database(&logctx.log).await; - let (opctx, datastore) = - datastore_test(&logctx.log, &db, Uuid::new_v4()).await; + let db = TestDatabase::new_with_datastore(&logctx.log).await; + let (opctx, datastore) = (db.opctx(), db.datastore()); let blueprint = BlueprintBuilder::build_empty_with_sleds( iter::once(SledUuid::new_v4()), @@ -573,8 +566,7 @@ mod tests { assert!(crdb_err3.contains(&crdb_zone_id3)); assert!(crdb_err3.contains(&crdb_zone_id4)); - datastore.terminate().await; - db.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } } diff --git a/nexus/src/app/background/tasks/networking.rs b/nexus/src/app/background/tasks/networking.rs index 95005d07cd..ed27409c9b 100644 --- a/nexus/src/app/background/tasks/networking.rs +++ b/nexus/src/app/background/tasks/networking.rs @@ -4,7 +4,7 @@ use db::datastore::SwitchPortSettingsCombinedResult; use dpd_client::types::{ - LinkCreate, LinkId, LinkSettings, PortFec, PortSettings, PortSpeed, + LinkCreate, LinkId, LinkSettings, PortFec, PortSettings, PortSpeed, TxEq, }; use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::db; @@ -63,7 +63,17 @@ pub(crate) fn api_to_dpd_port_settings( //TODO breakouts let link_id = LinkId(0); - + let tx_eq = if let Some(Some(t)) = settings.tx_eq.get(0) { + Some(TxEq { + pre1: t.pre1.map(Into::into), + pre2: t.pre2.map(Into::into), + main: t.main.map(Into::into), + post2: t.post2.map(Into::into), + post1: t.post2.map(Into::into), + }) + } else { + None + }; for l in settings.links.iter() { dpd_port_settings.links.insert( link_id.to_string(), @@ -72,6 +82,7 @@ pub(crate) fn api_to_dpd_port_settings( autoneg: l.autoneg, lane: Some(LinkId(0)), kr: false, + tx_eq: tx_eq.clone(), fec: match l.fec { SwitchLinkFec::Firecode => PortFec::Firecode, SwitchLinkFec::Rs => PortFec::Rs, diff --git a/nexus/src/app/background/tasks/saga_recovery.rs b/nexus/src/app/background/tasks/saga_recovery.rs index bee8884fd4..ad32f854af 100644 --- a/nexus/src/app/background/tasks/saga_recovery.rs +++ b/nexus/src/app/background/tasks/saga_recovery.rs @@ -484,10 +484,9 @@ mod test { use super::*; use nexus_auth::authn; use nexus_db_queries::context::OpContext; + use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_db_queries::db::test_utils::UnpluggableCockroachDbSecStore; - use nexus_test_utils::{ - db::test_setup_database, resource_helpers::create_project, - }; + use nexus_test_utils::resource_helpers::create_project; use nexus_test_utils_macros::nexus_test; use nexus_types::internal_api::views::LastResult; use omicron_test_utils::dev::{ @@ -506,24 +505,6 @@ mod test { type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; - // Returns a Cockroach DB, as well as a "datastore" interface (which is the - // one more frequently used by Nexus). - // - // The caller is responsible for calling "cleanup().await" on the returned - // CockroachInstance - we would normally wrap this in a drop method, but it - // is async. - async fn new_db( - log: &slog::Logger, - ) -> (dev::db::CockroachInstance, Arc) { - let db = test_setup_database(&log).await; - let cfg = nexus_db_queries::db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(db::Pool::new_single_host(log, &cfg)); - let db_datastore = Arc::new( - db::DataStore::new(&log, Arc::clone(&pool), None).await.unwrap(), - ); - (db, db_datastore) - } - // The following is our "saga-under-test". It's a simple two-node operation // that tracks how many times it has been called, and provides a mechanism // for detaching storage to simulate power failure (and meaningfully @@ -670,7 +651,8 @@ mod test { let logctx = dev::test_setup_log("test_failure_during_saga_can_be_recovered"); let log = logctx.log.new(o!()); - let (mut db, db_datastore) = new_db(&log).await; + let db = TestDatabase::new_with_raw_datastore(&log).await; + let db_datastore = db.datastore(); let sec_id = db::SecId(uuid::Uuid::new_v4()); let (storage, sec_client, uctx) = create_storage_sec_and_context(&log, db_datastore.clone(), sec_id); @@ -743,8 +725,7 @@ mod test { drop(task); let sec_client = Arc::try_unwrap(sec_client).unwrap(); sec_client.shutdown().await; - db_datastore.terminate().await; - db.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } @@ -757,7 +738,8 @@ mod test { "test_successful_saga_does_not_replay_during_recovery", ); let log = logctx.log.new(o!()); - let (mut db, db_datastore) = new_db(&log).await; + let db = TestDatabase::new_with_raw_datastore(&log).await; + let db_datastore = db.datastore(); let sec_id = db::SecId(uuid::Uuid::new_v4()); let (storage, sec_client, uctx) = create_storage_sec_and_context(&log, db_datastore.clone(), sec_id); @@ -814,8 +796,7 @@ mod test { drop(task); let sec_client = Arc::try_unwrap(sec_client).unwrap(); sec_client.shutdown().await; - db_datastore.terminate().await; - db.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } diff --git a/nexus/src/app/background/tasks/sync_switch_configuration.rs b/nexus/src/app/background/tasks/sync_switch_configuration.rs index 49dbfb2e52..ceacae7645 100644 --- a/nexus/src/app/background/tasks/sync_switch_configuration.rs +++ b/nexus/src/app/background/tasks/sync_switch_configuration.rs @@ -53,7 +53,7 @@ use sled_agent_client::types::{ BgpConfig as SledBgpConfig, BgpPeerConfig as SledBgpPeerConfig, EarlyNetworkConfig, EarlyNetworkConfigBody, HostPortConfig, LldpAdminStatus, LldpPortConfig, PortConfigV2, RackNetworkConfigV2, - RouteConfig as SledRouteConfig, UplinkAddressConfig, + RouteConfig as SledRouteConfig, TxEqConfig, UplinkAddressConfig, }; use std::{ collections::{hash_map::Entry, HashMap, HashSet}, @@ -935,6 +935,19 @@ impl BackgroundTask for SwitchPortSettingsManager { }, }; + // TODO https://github.com/oxidecomputer/omicron/issues/3062 + let tx_eq = if let Some(Some(c)) = info.tx_eq.get(0) { + Some(TxEqConfig { + pre1: c.pre1.map(Into::into), + pre2: c.pre2.map(Into::into), + main: c.main.map(Into::into), + post2: c.post2.map(Into::into), + post1: c.post1.map(Into::into), + }) + } else { + None + }; + let mut port_config = PortConfigV2 { addresses: info.addresses.iter().map(|a| UplinkAddressConfig { @@ -1012,7 +1025,8 @@ impl BackgroundTask for SwitchPortSettingsManager { system_name: c.system_name.clone(), system_description: c.system_description.clone(), management_addrs:c.management_ip.map(|a| vec![a.ip()]), - }) + }), + tx_eq, } ; @@ -1456,6 +1470,18 @@ fn uplinks( }) }; + let tx_eq = if let Some(Some(c)) = config.tx_eq.get(0) { + Some(TxEqConfig { + pre1: c.pre1, + pre2: c.pre2, + main: c.main, + post2: c.post2, + post1: c.post1, + }) + } else { + None + }; + let config = HostPortConfig { port: port.port_name.clone(), addrs: config @@ -1467,6 +1493,7 @@ fn uplinks( }) .collect(), lldp, + tx_eq, }; match uplinks.entry(*location) { diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 9a361d564d..a6e38e8dcf 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -650,12 +650,15 @@ impl super::Nexus { }, }, }; + let link = LinkConfigCreate { - mtu: 1500, //TODO https://github.com/oxidecomputer/omicron/issues/2274 + //TODO https://github.com/oxidecomputer/omicron/issues/2274 + mtu: 1500, fec: uplink_config.uplink_port_fec.into(), speed: uplink_config.uplink_port_speed.into(), autoneg: uplink_config.autoneg, lldp, + tx_eq: uplink_config.tx_eq.map(|t| t.into()), }; port_settings_params.links.insert("phy".to_string(), link); diff --git a/nexus/src/app/sagas/common_storage/pantry_pool.rs b/nexus/src/app/sagas/common_storage/pantry_pool.rs index 9d1e76d27d..ae0ee3ce2a 100644 --- a/nexus/src/app/sagas/common_storage/pantry_pool.rs +++ b/nexus/src/app/sagas/common_storage/pantry_pool.rs @@ -84,9 +84,12 @@ impl backend::Connector for PantryConnector { pub(crate) fn make_pantry_connection_pool( qorb_resolver: &QorbResolver, ) -> pool::Pool { - pool::Pool::new( + match pool::Pool::new( qorb_resolver.for_service(ServiceName::CruciblePantry), Arc::new(PantryConnector), qorb::policy::Policy::default(), - ) + ) { + Ok(pool) => pool, + Err(e) => e.into_inner(), + } } diff --git a/nexus/src/populate.rs b/nexus/src/populate.rs index 08713bcc25..0438a53353 100644 --- a/nexus/src/populate.rs +++ b/nexus/src/populate.rs @@ -357,7 +357,7 @@ mod test { use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; - use nexus_test_utils::db::test_setup_database; + use nexus_db_queries::db::pub_test_utils::TestDatabase; use omicron_common::api::external::Error; use omicron_test_utils::dev; use std::sync::Arc; @@ -378,8 +378,8 @@ mod test { p: &dyn Populator, ) { let logctx = dev::test_setup_log("test_populator"); - let mut db = test_setup_database(&logctx.log).await; - let cfg = db::Config { url: db.pg_config().clone() }; + let db = TestDatabase::new_populate_schema_only(&logctx.log).await; + let cfg = db::Config { url: db.crdb().pg_config().clone() }; let pool = Arc::new(db::Pool::new_single_host(&logctx.log, &cfg)); let datastore = Arc::new( db::DataStore::new(&logctx.log, pool, None).await.unwrap(), @@ -443,7 +443,7 @@ mod test { ); info!(&log, "cleaning up database"); - db.cleanup().await.unwrap(); + db.terminate().await; info!(&log, "populator {:?}, with database offline", p); match p.populate(&opctx, &datastore, &args).await { diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index c22a03a2f9..3756c6f86a 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -29,7 +29,7 @@ internal-dns-resolver.workspace = true internal-dns-types.workspace = true nexus-client.workspace = true nexus-config.workspace = true -nexus-db-queries.workspace = true +nexus-db-queries = { workspace = true, features = [ "testing" ] } nexus-sled-agent-shared.workspace = true nexus-test-interface.workspace = true nexus-types.workspace = true diff --git a/nexus/test-utils/src/db.rs b/nexus/test-utils/src/db.rs index ff23f35df0..6f3ffc8e67 100644 --- a/nexus/test-utils/src/db.rs +++ b/nexus/test-utils/src/db.rs @@ -4,64 +4,6 @@ //! Database testing facilities. -use camino::Utf8PathBuf; -use omicron_test_utils::dev; -use slog::Logger; - -/// Path to the "seed" CockroachDB tarball. -/// -/// Populating CockroachDB unfortunately isn't free - creation of -/// tables, indices, and users takes several seconds to complete. -/// -/// By creating a "seed" version of the database, we can cut down -/// on the time spent performing this operation. Instead, we opt -/// to copy the database from this seed location. -fn seed_tar() -> Utf8PathBuf { - // The setup script should set this environment variable. - let seed_dir = std::env::var(dev::CRDB_SEED_TAR_ENV).unwrap_or_else(|_| { - panic!( - "{} missing -- are you running this test \ - with `cargo nextest run`?", - dev::CRDB_SEED_TAR_ENV, - ) - }); - seed_dir.into() -} - -/// Wrapper around [`dev::test_setup_database`] which uses a seed tarball -/// provided from the environment. -pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance { - let input_tar = seed_tar(); - dev::test_setup_database( - log, - dev::StorageSource::CopyFromSeed { input_tar }, - ) - .await -} - -/// Wrapper around [`dev::test_setup_database`] which uses a seed tarball -/// provided as an argument. -#[cfg(feature = "omicron-dev")] -pub async fn test_setup_database_from_seed( - log: &Logger, - input_tar: Utf8PathBuf, -) -> dev::db::CockroachInstance { - dev::test_setup_database( - log, - dev::StorageSource::CopyFromSeed { input_tar }, - ) - .await -} - -/// Creates a new database with no data populated. -/// -/// Primarily used for schema change and migration testing. -pub async fn test_setup_database_empty( - log: &Logger, -) -> dev::db::CockroachInstance { - dev::test_setup_database(log, dev::StorageSource::DoNotPopulate).await -} - /// See the definition of this constant in nexus_db_queries. /// /// Besides the cases mentioned there, it's also preferable for some ad hoc @@ -69,3 +11,9 @@ pub async fn test_setup_database_empty( /// used for the test suite. pub const ALLOW_FULL_TABLE_SCAN_SQL: &str = nexus_db_queries::db::queries::ALLOW_FULL_TABLE_SCAN_SQL; + +/// Refer to nexus_db_queries for additional documentation. +/// +/// This structure enables callers to construct an arbitrarily +/// populated database, with or without a datastore on top. +pub use nexus_db_queries::db::pub_test_utils::TestDatabase; diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index c74e2faf6b..0c5df8c844 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -31,6 +31,7 @@ use nexus_config::InternalDns; use nexus_config::MgdConfig; use nexus_config::NexusConfig; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; +use nexus_db_queries::db::pub_test_utils::crdb; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_sled_agent_shared::inventory::OmicronZonesConfig; use nexus_sled_agent_shared::recovery_silo::RecoverySiloConfig; @@ -94,7 +95,7 @@ pub mod resource_helpers; pub const SLED_AGENT_UUID: &str = "b6d65341-167c-41df-9b5c-41cded99c229"; pub const SLED_AGENT2_UUID: &str = "039be560-54cc-49e3-88df-1a29dadbf913"; -pub const RACK_UUID: &str = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"; +pub const RACK_UUID: &str = nexus_db_queries::db::pub_test_utils::RACK_UUID; pub const SWITCH_UUID: &str = "dae4e1f1-410e-4314-bff1-fec0504be07e"; pub const PHYSICAL_DISK_UUID: &str = "fbf4e1f1-410e-4314-bff1-fec0504be07e"; pub const OXIMETER_UUID: &str = "39e6175b-4df2-4730-b11d-cbc1e60a2e78"; @@ -420,13 +421,13 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { // Start up CockroachDB. let database = match populate { PopulateCrdb::FromEnvironmentSeed => { - db::test_setup_database(log).await + crdb::test_setup_database(log).await } #[cfg(feature = "omicron-dev")] PopulateCrdb::FromSeed { input_tar } => { - db::test_setup_database_from_seed(log, input_tar).await + crdb::test_setup_database_from_seed(log, input_tar).await } - PopulateCrdb::Empty => db::test_setup_database_empty(log).await, + PopulateCrdb::Empty => crdb::test_setup_database_empty(log).await, }; eprintln!("DB URL: {}", database.pg_config()); @@ -828,6 +829,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { // // However, for now, this isn't necessary. blueprint_disks: BTreeMap::new(), + blueprint_datasets: BTreeMap::new(), sled_state, parent_blueprint_id: None, internal_dns_version: dns_config.generation, diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index f852312de2..fa41d17aa9 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -11,8 +11,9 @@ use nexus_config::SchemaConfig; use nexus_db_model::EARLIEST_SUPPORTED_VERSION; use nexus_db_model::SCHEMA_VERSION as LATEST_SCHEMA_VERSION; use nexus_db_model::{AllSchemaVersions, SchemaVersion}; +use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_db_queries::db::DISALLOW_FULL_TABLE_SCAN_SQL; -use nexus_test_utils::{db, load_test_config, ControlPlaneTestContextBuilder}; +use nexus_test_utils::{load_test_config, ControlPlaneTestContextBuilder}; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::shared::SwitchLocation; use omicron_test_utils::dev::db::{Client, CockroachInstance}; @@ -28,19 +29,6 @@ use uuid::Uuid; const SCHEMA_DIR: &'static str = concat!(env!("CARGO_MANIFEST_DIR"), "/../schema/crdb"); -async fn test_setup_just_crdb<'a>( - log: &Logger, - populate: bool, -) -> CockroachInstance { - // Start up CockroachDB. - let database = if populate { - db::test_setup_database(log).await - } else { - db::test_setup_database_empty(log).await - }; - database -} - // Helper to ensure we perform the same setup for the positive and negative test // cases. async fn test_setup<'a>( @@ -538,15 +526,15 @@ async fn dbinit_version_matches_version_known_to_nexus() { &config.pkg.log, ); let log = &logctx.log; - let populate = true; - let mut crdb = test_setup_just_crdb(&log, populate).await; + let db = TestDatabase::new_populate_schema_only(&log).await; + let crdb = db.crdb(); assert_eq!( LATEST_SCHEMA_VERSION.to_string(), - query_crdb_schema_version(&crdb).await + query_crdb_schema_version(crdb).await ); - crdb.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } @@ -611,8 +599,8 @@ async fn versions_have_idempotent_up() { let logctx = LogContext::new("versions_have_idempotent_up", &config.pkg.log); let log = &logctx.log; - let populate = false; - let mut crdb = test_setup_just_crdb(&logctx.log, populate).await; + let db = TestDatabase::new_populate_nothing(&logctx.log).await; + let crdb = db.crdb(); let all_versions = read_all_schema_versions(); for version in all_versions.iter_versions() { @@ -627,7 +615,7 @@ async fn versions_have_idempotent_up() { query_crdb_schema_version(&crdb).await ); - crdb.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } @@ -940,8 +928,8 @@ async fn dbinit_equals_sum_of_all_up() { LogContext::new("dbinit_equals_sum_of_all_up", &config.pkg.log); let log = &logctx.log; - let populate = false; - let mut crdb = test_setup_just_crdb(&logctx.log, populate).await; + let db = TestDatabase::new_populate_nothing(&logctx.log).await; + let crdb = db.crdb(); let all_versions = read_all_schema_versions(); @@ -1014,11 +1002,11 @@ async fn dbinit_equals_sum_of_all_up() { std::mem::drop(conn_from_pool); pool.terminate().await; std::mem::drop(pool); - crdb.cleanup().await.unwrap(); + db.terminate().await; // Create a new DB with data populated from dbinit.sql for comparison - let populate = true; - let mut crdb = test_setup_just_crdb(&logctx.log, populate).await; + let db = TestDatabase::new_populate_schema_only(&logctx.log).await; + let crdb = db.crdb(); let expected_schema = InformationSchema::new(&crdb).await; let expected_data = expected_schema.query_all_tables(log, &crdb).await; @@ -1026,7 +1014,7 @@ async fn dbinit_equals_sum_of_all_up() { observed_schema.pretty_assert_eq(&expected_schema); assert_eq!(observed_data, expected_data); - crdb.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } @@ -1635,8 +1623,8 @@ async fn validate_data_migration() { let logctx = LogContext::new("validate_data_migration", &config.pkg.log); let log = &logctx.log; - let populate = false; - let mut crdb = test_setup_just_crdb(&logctx.log, populate).await; + let db = TestDatabase::new_populate_nothing(&logctx.log).await; + let crdb = db.crdb(); let client = crdb.connect().await.expect("Failed to access CRDB client"); let all_versions = read_all_schema_versions(); @@ -1666,20 +1654,20 @@ async fn validate_data_migration() { query_crdb_schema_version(&crdb).await ); - crdb.cleanup().await.unwrap(); + db.terminate().await; logctx.cleanup_successful(); } // Returns the InformationSchema object for a database populated via `sql`. async fn get_information_schema(log: &Logger, sql: &str) -> InformationSchema { - let populate = false; - let mut crdb = test_setup_just_crdb(&log, populate).await; + let db = TestDatabase::new_populate_nothing(&log).await; + let crdb = db.crdb(); let client = crdb.connect().await.expect("failed to connect"); client.batch_execute(sql).await.expect("failed to apply SQL"); let observed_schema = InformationSchema::new(&crdb).await; - crdb.cleanup().await.unwrap(); + db.terminate().await; observed_schema } diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index 6500d7249a..9ce686db56 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -130,6 +130,7 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { fec: LinkFec::None, speed: LinkSpeed::Speed100G, autoneg: false, + tx_eq: None, }, ); // interfaces diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index ef2f366a28..365fe3f19e 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -22,10 +22,17 @@ use nexus_sled_agent_shared::inventory::OmicronZoneConfig; use nexus_sled_agent_shared::inventory::OmicronZoneType; use nexus_sled_agent_shared::inventory::OmicronZonesConfig; use nexus_sled_agent_shared::inventory::ZoneKind; +use omicron_common::api::external::ByteCount; use omicron_common::api::external::Generation; +use omicron_common::api::internal::shared::DatasetKind; +use omicron_common::disk::CompressionAlgorithm; +use omicron_common::disk::DatasetConfig; +use omicron_common::disk::DatasetName; +use omicron_common::disk::DatasetsConfig; use omicron_common::disk::DiskIdentity; use omicron_common::disk::OmicronPhysicalDisksConfig; use omicron_uuid_kinds::CollectionUuid; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; use schemars::JsonSchema; @@ -35,6 +42,7 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt; use std::net::Ipv6Addr; +use std::net::SocketAddrV6; use strum::EnumIter; use strum::IntoEnumIterator; use uuid::Uuid; @@ -149,6 +157,9 @@ pub struct Blueprint { /// A map of sled id -> disks in use on each sled. pub blueprint_disks: BTreeMap, + /// A map of sled id -> datasets in use on each sled + pub blueprint_datasets: BTreeMap, + /// which blueprint this blueprint is based on pub parent_blueprint_id: Option, @@ -226,6 +237,17 @@ impl Blueprint { }) } + /// Iterate over the [`BlueprintDatasetsConfig`] instances in the blueprint. + pub fn all_omicron_datasets( + &self, + filter: BlueprintDatasetFilter, + ) -> impl Iterator { + self.blueprint_datasets + .iter() + .flat_map(move |(_, datasets)| datasets.datasets.values()) + .filter(move |d| d.disposition.matches(filter)) + } + /// Iterate over the [`BlueprintZoneConfig`] instances in the blueprint /// that do not match the provided filter, along with the associated sled /// id. @@ -800,6 +822,22 @@ pub enum BlueprintZoneFilter { ShouldDeployVpcFirewallRules, } +/// Filters that apply to blueprint datasets. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum BlueprintDatasetFilter { + // --- + // Prefer to keep this list in alphabetical order. + // --- + /// All datasets + All, + + /// Datasets that have been expunged. + Expunged, + + /// Datasets that are in-service. + InService, +} + /// Information about an Omicron physical disk as recorded in a blueprint. /// /// Part of [`Blueprint`]. @@ -809,6 +847,95 @@ pub type BlueprintPhysicalDisksConfig = pub type BlueprintPhysicalDiskConfig = omicron_common::disk::OmicronPhysicalDiskConfig; +/// Information about Omicron datasets as recorded in a blueprint. +#[derive(Debug, Clone, Eq, PartialEq, JsonSchema, Deserialize, Serialize)] +pub struct BlueprintDatasetsConfig { + pub generation: Generation, + pub datasets: BTreeMap, +} + +impl From for DatasetsConfig { + fn from(config: BlueprintDatasetsConfig) -> Self { + Self { + generation: config.generation, + datasets: config + .datasets + .into_iter() + .map(|(id, d)| (id, d.into())) + .collect(), + } + } +} + +/// The desired state of an Omicron-managed dataset in a blueprint. +/// +/// Part of [`BlueprintDatasetConfig`]. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + JsonSchema, + Deserialize, + Serialize, + EnumIter, +)] +#[serde(rename_all = "snake_case")] +pub enum BlueprintDatasetDisposition { + /// The dataset is in-service. + InService, + + /// The dataset is permanently gone. + Expunged, +} + +impl BlueprintDatasetDisposition { + pub fn matches(self, filter: BlueprintDatasetFilter) -> bool { + match self { + Self::InService => match filter { + BlueprintDatasetFilter::All => true, + BlueprintDatasetFilter::Expunged => false, + BlueprintDatasetFilter::InService => true, + }, + Self::Expunged => match filter { + BlueprintDatasetFilter::All => true, + BlueprintDatasetFilter::Expunged => true, + BlueprintDatasetFilter::InService => false, + }, + } + } +} + +/// Information about a dataset as recorded in a blueprint +#[derive(Debug, Clone, Eq, PartialEq, JsonSchema, Deserialize, Serialize)] +pub struct BlueprintDatasetConfig { + pub disposition: BlueprintDatasetDisposition, + + pub id: DatasetUuid, + pub pool: ZpoolName, + pub kind: DatasetKind, + pub address: Option, + pub quota: Option, + pub reservation: Option, + pub compression: CompressionAlgorithm, +} + +impl From for DatasetConfig { + fn from(config: BlueprintDatasetConfig) -> Self { + Self { + id: config.id, + name: DatasetName::new(config.pool, config.kind), + quota: config.quota, + reservation: config.reservation, + compression: config.compression, + } + } +} + /// Describe high-level metadata about a blueprint // These fields are a subset of [`Blueprint`], and include only the data we can // quickly fetch from the main blueprint table (e.g., when listing all diff --git a/nexus/types/src/deployment/execution/spec.rs b/nexus/types/src/deployment/execution/spec.rs index 4b64477bf2..2472a47d03 100644 --- a/nexus/types/src/deployment/execution/spec.rs +++ b/nexus/types/src/deployment/execution/spec.rs @@ -33,6 +33,7 @@ pub enum ExecutionComponent { OmicronZones, FirewallRules, DatasetRecords, + Datasets, Dns, Cockroach, Clickhouse, diff --git a/nexus/types/src/deployment/planning_input.rs b/nexus/types/src/deployment/planning_input.rs index ea23465183..57cc99709e 100644 --- a/nexus/types/src/deployment/planning_input.rs +++ b/nexus/types/src/deployment/planning_input.rs @@ -23,6 +23,7 @@ use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::SourceNatConfigError; +use omicron_common::disk::DatasetConfig; use omicron_common::disk::DiskIdentity; use omicron_common::policy::SINGLE_NODE_CLICKHOUSE_REDUNDANCY; use omicron_uuid_kinds::OmicronZoneUuid; @@ -604,7 +605,7 @@ pub struct SledResources { /// storage) // NOTE: I'd really like to make this private, to make it harder to // accidentally pick a zpool that is not in-service. - pub zpools: BTreeMap, + pub zpools: BTreeMap)>, /// the IPv6 subnet of this sled on the underlay network /// @@ -616,7 +617,9 @@ pub struct SledResources { impl SledResources { /// Returns if the zpool is provisionable (known, in-service, and active). pub fn zpool_is_provisionable(&self, zpool: &ZpoolUuid) -> bool { - let Some(disk) = self.zpools.get(zpool) else { return false }; + let Some((disk, _datasets)) = self.zpools.get(zpool) else { + return false; + }; disk.provisionable() } @@ -625,7 +628,7 @@ impl SledResources { &self, filter: ZpoolFilter, ) -> impl Iterator + '_ { - self.zpools.iter().filter_map(move |(zpool, disk)| { + self.zpools.iter().filter_map(move |(zpool, (disk, _datasets))| { filter .matches_policy_and_state(disk.policy, disk.state) .then_some(zpool) @@ -636,12 +639,23 @@ impl SledResources { &self, filter: DiskFilter, ) -> impl Iterator + '_ { - self.zpools.iter().filter_map(move |(zpool, disk)| { + self.zpools.iter().filter_map(move |(zpool, (disk, _datasets))| { filter .matches_policy_and_state(disk.policy, disk.state) .then_some((zpool, disk)) }) } + + pub fn all_datasets( + &self, + filter: ZpoolFilter, + ) -> impl Iterator + '_ { + self.zpools.iter().filter_map(move |(zpool, (disk, datasets))| { + filter + .matches_policy_and_state(disk.policy, disk.state) + .then_some((zpool, datasets.as_slice())) + }) + } } /// Filters that apply to sleds. diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 4daf61d4d0..9af9d3be1e 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -14,7 +14,7 @@ use omicron_common::api::external::{ IdentityMetadataCreateParams, IdentityMetadataUpdateParams, InstanceAutoRestartPolicy, InstanceCpuCount, LinkFec, LinkSpeed, Name, NameOrId, PaginationOrder, RouteDestination, RouteTarget, SemverVersion, - UserId, + TxEqConfig, UserId, }; use omicron_common::disk::DiskVariant; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; @@ -1137,9 +1137,18 @@ pub struct InstanceCreate { /// The auto-restart policy for this instance. /// - /// This indicates whether the instance should be automatically restarted by - /// the control plane on failure. If this is `null`, no auto-restart policy - /// has been configured for this instance by the user. + /// This policy determines whether the instance should be automatically + /// restarted by the control plane on failure. If this is `null`, no + /// auto-restart policy will be explicitly configured for this instance, and + /// the control plane will select the default policy when determining + /// whether the instance can be automatically restarted. + /// + /// Currently, the global default auto-restart policy is "best-effort", so + /// instances with `null` auto-restart policies will be automatically + /// restarted. However, in the future, the default policy may be + /// configurable through other mechanisms, such as on a per-project basis. + /// In that case, any configured default policy will be used if this is + /// `null`. #[serde(default)] pub auto_restart_policy: Option, } @@ -1152,9 +1161,20 @@ pub struct InstanceUpdate { /// If not provided, unset the instance's boot disk. pub boot_disk: Option, - /// The auto-restart policy for this instance. + /// Sets the auto-restart policy for this instance. /// - /// If not provided, unset the instance's auto-restart policy. + /// This policy determines whether the instance should be automatically + /// restarted by the control plane on failure. If this is `null`, any + /// explicitly configured auto-restart policy will be unset, and + /// the control plane will select the default policy when determining + /// whether the instance can be automatically restarted. + /// + /// Currently, the global default auto-restart policy is "best-effort", so + /// instances with `null` auto-restart policies will be automatically + /// restarted. However, in the future, the default policy may be + /// configurable through other mechanisms, such as on a per-project basis. + /// In that case, any configured default policy will be used if this is + /// `null`. pub auto_restart_policy: Option, } @@ -1703,6 +1723,9 @@ pub struct LinkConfigCreate { /// Whether or not to set autonegotiation pub autoneg: bool, + + /// Optional tx_eq settings + pub tx_eq: Option, } /// The LLDP configuration associated with a port. diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 54feaa7325..2f3058cd46 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -856,6 +856,15 @@ } ] }, + "tx_eq": { + "nullable": true, + "description": "TX-EQ configuration for this port", + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] + }, "uplink_port_fec": { "description": "Port forward error correction type.", "allOf": [ @@ -1536,6 +1545,42 @@ } ] }, + "TxEqConfig": { + "description": "Per-port tx-eq overrides. This can be used to fine-tune the transceiver equalization settings to improve signal integrity.", + "type": "object", + "properties": { + "main": { + "nullable": true, + "description": "Main tap", + "type": "integer", + "format": "int32" + }, + "post1": { + "nullable": true, + "description": "Post-cursor tap1", + "type": "integer", + "format": "int32" + }, + "post2": { + "nullable": true, + "description": "Post-cursor tap2", + "type": "integer", + "format": "int32" + }, + "pre1": { + "nullable": true, + "description": "Pre-cursor tap1", + "type": "integer", + "format": "int32" + }, + "pre2": { + "nullable": true, + "description": "Pre-cursor tap2", + "type": "integer", + "format": "int32" + } + } + }, "TypedUuidForRackInitKind": { "type": "string", "format": "uuid" diff --git a/openapi/clickhouse-admin-server.json b/openapi/clickhouse-admin-server.json index 9dc2506eb1..52faddb3b9 100644 --- a/openapi/clickhouse-admin-server.json +++ b/openapi/clickhouse-admin-server.json @@ -44,6 +44,35 @@ } } } + }, + "/distributed-ddl-queue": { + "get": { + "summary": "Contains information about distributed ddl queries (ON CLUSTER clause)", + "description": "that were executed on a cluster.", + "operationId": "distributed_ddl_queue", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_DistributedDdlQueue", + "type": "array", + "items": { + "$ref": "#/components/schemas/DistributedDdlQueue" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } } }, "components": { @@ -90,6 +119,101 @@ } ] }, + "DistributedDdlQueue": { + "description": "Contains information about distributed ddl queries (ON CLUSTER clause) that were executed on a cluster.", + "type": "object", + "properties": { + "cluster": { + "description": "Cluster name", + "type": "string" + }, + "entry": { + "description": "Query id", + "type": "string" + }, + "entry_version": { + "description": "Version of the entry", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "exception_code": { + "description": "Exception code", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "exception_text": { + "description": "Exception message", + "type": "string" + }, + "host": { + "description": "Hostname", + "type": "string", + "format": "ipv6" + }, + "initiator_host": { + "description": "Host that initiated the DDL operation", + "type": "string" + }, + "initiator_port": { + "description": "Port used by the initiator", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "port": { + "description": "Host Port", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "query": { + "description": "Query executed", + "type": "string" + }, + "query_create_time": { + "description": "Query created time", + "type": "string" + }, + "query_duration_ms": { + "description": "Duration of query execution (in milliseconds)", + "type": "string" + }, + "query_finish_time": { + "description": "Query finish time", + "type": "string" + }, + "settings": { + "description": "Settings used in the DDL operation", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "status": { + "description": "Status of the query", + "type": "string" + } + }, + "required": [ + "cluster", + "entry", + "entry_version", + "exception_code", + "exception_text", + "host", + "initiator_host", + "initiator_port", + "port", + "query", + "query_create_time", + "query_duration_ms", + "query_finish_time", + "settings", + "status" + ] + }, "Error": { "description": "Error information from a response.", "type": "object", diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 2d1465d2a7..4392de318d 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -1907,6 +1907,13 @@ "description": "Describes a complete set of software and configuration for the system", "type": "object", "properties": { + "blueprint_datasets": { + "description": "A map of sled id -> datasets in use on each sled", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/BlueprintDatasetsConfig" + } + }, "blueprint_disks": { "description": "A map of sled id -> disks in use on each sled.", "type": "object", @@ -1991,6 +1998,7 @@ } }, "required": [ + "blueprint_datasets", "blueprint_disks", "blueprint_zones", "cockroachdb_fingerprint", @@ -2004,6 +2012,92 @@ "time_created" ] }, + "BlueprintDatasetConfig": { + "description": "Information about a dataset as recorded in a blueprint", + "type": "object", + "properties": { + "address": { + "nullable": true, + "type": "string" + }, + "compression": { + "$ref": "#/components/schemas/CompressionAlgorithm" + }, + "disposition": { + "$ref": "#/components/schemas/BlueprintDatasetDisposition" + }, + "id": { + "$ref": "#/components/schemas/TypedUuidForDatasetKind" + }, + "kind": { + "$ref": "#/components/schemas/DatasetKind" + }, + "pool": { + "$ref": "#/components/schemas/ZpoolName" + }, + "quota": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + }, + "reservation": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/ByteCount" + } + ] + } + }, + "required": [ + "compression", + "disposition", + "id", + "kind", + "pool" + ] + }, + "BlueprintDatasetDisposition": { + "description": "The desired state of an Omicron-managed dataset in a blueprint.\n\nPart of [`BlueprintDatasetConfig`].", + "oneOf": [ + { + "description": "The dataset is in-service.", + "type": "string", + "enum": [ + "in_service" + ] + }, + { + "description": "The dataset is permanently gone.", + "type": "string", + "enum": [ + "expunged" + ] + } + ] + }, + "BlueprintDatasetsConfig": { + "description": "Information about Omicron datasets as recorded in a blueprint.", + "type": "object", + "properties": { + "datasets": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/BlueprintDatasetConfig" + } + }, + "generation": { + "$ref": "#/components/schemas/Generation" + } + }, + "required": [ + "datasets", + "generation" + ] + }, "BlueprintMetadata": { "description": "Describe high-level metadata about a blueprint", "type": "object", @@ -2830,6 +2924,112 @@ } ] }, + "CompressionAlgorithm": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "on" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "off" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "gzip" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "level": { + "$ref": "#/components/schemas/GzipLevel" + }, + "type": { + "type": "string", + "enum": [ + "gzip_n" + ] + } + }, + "required": [ + "level", + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lz4" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lzjb" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "zle" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, "CurrentStatus": { "description": "Describes the current status of a background task", "oneOf": [ @@ -3475,6 +3675,11 @@ "format": "uint64", "minimum": 0 }, + "GzipLevel": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, "ImportExportPolicy": { "description": "Define policy relating to the import and export of prefixes from a BGP peer.", "oneOf": [ @@ -3532,7 +3737,7 @@ }, "auto_restart_policy": { "nullable": true, - "description": "The auto-restart policy configured for this instance, or `None` if no explicit policy is configured.\n\nIf this is not present, then this instance uses the default auto-restart policy, which may or may not allow it to be restarted. The `auto_restart_enabled` field indicates whether the instance will be automatically restarted.", + "description": "The auto-restart policy configured for this instance, or `null` if no explicit policy has been configured.\n\nThis policy determines whether the instance should be automatically restarted by the control plane on failure. If this is `null`, the control plane will use the default policy when determining whether or not to automatically restart this instance, which may or may not allow it to be restarted. The value of the `auto_restart_enabled` field indicates whether the instance will be auto-restarted, based on its current policy or the default if it has no configured policy.", "allOf": [ { "$ref": "#/components/schemas/InstanceAutoRestartPolicy" @@ -4472,6 +4677,15 @@ } ] }, + "tx_eq": { + "nullable": true, + "description": "TX-EQ configuration for this port", + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] + }, "uplink_port_fec": { "description": "Port forward error correction type.", "allOf": [ @@ -5541,6 +5755,46 @@ "SwitchPutResponse": { "type": "object" }, + "TxEqConfig": { + "description": "Per-port tx-eq overrides. This can be used to fine-tune the transceiver equalization settings to improve signal integrity.", + "type": "object", + "properties": { + "main": { + "nullable": true, + "description": "Main tap", + "type": "integer", + "format": "int32" + }, + "post1": { + "nullable": true, + "description": "Post-cursor tap1", + "type": "integer", + "format": "int32" + }, + "post2": { + "nullable": true, + "description": "Post-cursor tap2", + "type": "integer", + "format": "int32" + }, + "pre1": { + "nullable": true, + "description": "Pre-cursor tap1", + "type": "integer", + "format": "int32" + }, + "pre2": { + "nullable": true, + "description": "Pre-cursor tap2", + "type": "integer", + "format": "int32" + } + } + }, + "TypedUuidForDatasetKind": { + "type": "string", + "format": "uuid" + }, "TypedUuidForDemoSagaKind": { "type": "string", "format": "uuid" diff --git a/openapi/nexus.json b/openapi/nexus.json index 97f9830b7a..f12ff4730c 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -15816,7 +15816,7 @@ }, "auto_restart_policy": { "nullable": true, - "description": "The auto-restart policy configured for this instance, or `None` if no explicit policy is configured.\n\nIf this is not present, then this instance uses the default auto-restart policy, which may or may not allow it to be restarted. The `auto_restart_enabled` field indicates whether the instance will be automatically restarted.", + "description": "The auto-restart policy configured for this instance, or `null` if no explicit policy has been configured.\n\nThis policy determines whether the instance should be automatically restarted by the control plane on failure. If this is `null`, the control plane will use the default policy when determining whether or not to automatically restart this instance, which may or may not allow it to be restarted. The value of the `auto_restart_enabled` field indicates whether the instance will be auto-restarted, based on its current policy or the default if it has no configured policy.", "allOf": [ { "$ref": "#/components/schemas/InstanceAutoRestartPolicy" @@ -15941,7 +15941,7 @@ "properties": { "auto_restart_policy": { "nullable": true, - "description": "The auto-restart policy for this instance.\n\nThis indicates whether the instance should be automatically restarted by the control plane on failure. If this is `null`, no auto-restart policy has been configured for this instance by the user.", + "description": "The auto-restart policy for this instance.\n\nThis policy determines whether the instance should be automatically restarted by the control plane on failure. If this is `null`, no auto-restart policy will be explicitly configured for this instance, and the control plane will select the default policy when determining whether the instance can be automatically restarted.\n\nCurrently, the global default auto-restart policy is \"best-effort\", so instances with `null` auto-restart policies will be automatically restarted. However, in the future, the default policy may be configurable through other mechanisms, such as on a per-project basis. In that case, any configured default policy will be used if this is `null`.", "default": null, "allOf": [ { @@ -16476,7 +16476,7 @@ "properties": { "auto_restart_policy": { "nullable": true, - "description": "The auto-restart policy for this instance.\n\nIf not provided, unset the instance's auto-restart policy.", + "description": "Sets the auto-restart policy for this instance.\n\nThis policy determines whether the instance should be automatically restarted by the control plane on failure. If this is `null`, any explicitly configured auto-restart policy will be unset, and the control plane will select the default policy when determining whether the instance can be automatically restarted.\n\nCurrently, the global default auto-restart policy is \"best-effort\", so instances with `null` auto-restart policies will be automatically restarted. However, in the future, the default policy may be configurable through other mechanisms, such as on a per-project basis. In that case, any configured default policy will be used if this is `null`.", "allOf": [ { "$ref": "#/components/schemas/InstanceAutoRestartPolicy" @@ -17194,6 +17194,15 @@ "$ref": "#/components/schemas/LinkSpeed" } ] + }, + "tx_eq": { + "nullable": true, + "description": "Optional tx_eq settings", + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] } }, "required": [ @@ -20436,6 +20445,12 @@ "$ref": "#/components/schemas/LinkSpeed" } ] + }, + "tx_eq_config_id": { + "nullable": true, + "description": "The tx_eq configuration id for this link.", + "type": "string", + "format": "uuid" } }, "required": [ @@ -20736,6 +20751,18 @@ } ] }, + "tx_eq": { + "description": "TX equalization settings. These are optional, and most links will not need them.", + "type": "array", + "items": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] + } + }, "vlan_interfaces": { "description": "Vlan interface settings.", "type": "array", @@ -20754,6 +20781,7 @@ "port", "routes", "settings", + "tx_eq", "vlan_interfaces" ] }, @@ -20940,6 +20968,42 @@ "items" ] }, + "TxEqConfig": { + "description": "Per-port tx-eq overrides. This can be used to fine-tune the transceiver equalization settings to improve signal integrity.", + "type": "object", + "properties": { + "main": { + "nullable": true, + "description": "Main tap", + "type": "integer", + "format": "int32" + }, + "post1": { + "nullable": true, + "description": "Post-cursor tap1", + "type": "integer", + "format": "int32" + }, + "post2": { + "nullable": true, + "description": "Post-cursor tap2", + "type": "integer", + "format": "int32" + }, + "pre1": { + "nullable": true, + "description": "Pre-cursor tap1", + "type": "integer", + "format": "int32" + }, + "pre2": { + "nullable": true, + "description": "Pre-cursor tap2", + "type": "integer", + "format": "int32" + } + } + }, "UninitializedSled": { "description": "A sled that has not been added to an initialized rack yet", "type": "object", diff --git a/openapi/repo-depot.json b/openapi/repo-depot.json new file mode 100644 index 0000000000..0c0019cf8d --- /dev/null +++ b/openapi/repo-depot.json @@ -0,0 +1,82 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "Oxide TUF Repo Depot API", + "description": "API for fetching update artifacts", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "0.0.1" + }, + "paths": { + "/artifact/sha256/{sha256}": { + "get": { + "summary": "Fetch an artifact from the depot.", + "operationId": "artifact_get_by_sha256", + "parameters": [ + { + "in": "path", + "name": "sha256", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "*/*": { + "schema": {} + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index fdeff639b8..9963526ef8 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -10,6 +10,149 @@ "version": "0.0.1" }, "paths": { + "/artifacts": { + "get": { + "operationId": "artifact_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Map_of_uint", + "type": "object", + "additionalProperties": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/artifacts/{sha256}": { + "put": { + "operationId": "artifact_put", + "parameters": [ + { + "in": "path", + "name": "sha256", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactPutResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "artifact_delete", + "parameters": [ + { + "in": "path", + "name": "sha256", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/artifacts/{sha256}/copy-from-depot": { + "post": { + "operationId": "artifact_copy_from_depot", + "parameters": [ + { + "in": "path", + "name": "sha256", + "required": true, + "schema": { + "type": "string", + "format": "hex string (32 bytes)" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactCopyFromDepotBody" + } + } + }, + "required": true + }, + "responses": { + "202": { + "description": "successfully enqueued operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ArtifactCopyFromDepotResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/boot-disk/{boot_disk}/os/write": { "post": { "summary": "Write a new host OS image to the specified boot disk", @@ -1417,6 +1560,41 @@ "start_request" ] }, + "ArtifactCopyFromDepotBody": { + "type": "object", + "properties": { + "depot_base_url": { + "type": "string" + } + }, + "required": [ + "depot_base_url" + ] + }, + "ArtifactCopyFromDepotResponse": { + "type": "object" + }, + "ArtifactPutResponse": { + "type": "object", + "properties": { + "datasets": { + "description": "The number of valid M.2 artifact datasets we found on the sled. There is typically one of these datasets for each functional M.2.", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "successful_writes": { + "description": "The number of valid writes to the M.2 artifact datasets. This should be less than or equal to the number of artifact datasets.", + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "datasets", + "successful_writes" + ] + }, "Baseboard": { "description": "Describes properties that should uniquely identify a Gimlet.", "oneOf": [ @@ -1883,11 +2061,11 @@ ] }, "BootOrderEntry": { - "description": "An entry in a list of boot options.\n\n
JSON schema\n\n```json { \"description\": \"An entry in a list of boot options.\", \"type\": \"object\", \"required\": [ \"name\" ], \"properties\": { \"name\": { \"description\": \"The name of the device to attempt booting from.\", \"type\": \"string\" } } } ```
", + "description": "An entry in the boot order stored in a [`BootSettings`] component.\n\n
JSON schema\n\n```json { \"description\": \"An entry in the boot order stored in a [`BootSettings`] component.\", \"type\": \"object\", \"required\": [ \"name\" ], \"properties\": { \"name\": { \"description\": \"The name of another component in the spec that Propolis should try to boot from.\\n\\nCurrently, only disk device components are supported.\", \"type\": \"string\" } } } ```
", "type": "object", "properties": { "name": { - "description": "The name of the device to attempt booting from.", + "description": "The name of another component in the spec that Propolis should try to boot from.\n\nCurrently, only disk device components are supported.", "type": "string" } }, @@ -1896,10 +2074,11 @@ ] }, "BootSettings": { - "description": "BootSettings\n\n
JSON schema\n\n```json { \"type\": \"object\", \"required\": [ \"order\" ], \"properties\": { \"order\": { \"type\": \"array\", \"items\": { \"$ref\": \"#/components/schemas/BootOrderEntry\" } } } } ```
", + "description": "Settings supplied to the guest's firmware image that specify the order in which it should consider its options when selecting a device to try to boot from.\n\n
JSON schema\n\n```json { \"description\": \"Settings supplied to the guest's firmware image that specify the order in which it should consider its options when selecting a device to try to boot from.\", \"type\": \"object\", \"required\": [ \"order\" ], \"properties\": { \"order\": { \"description\": \"An ordered list of components to attempt to boot from.\", \"type\": \"array\", \"items\": { \"$ref\": \"#/components/schemas/BootOrderEntry\" } } }, \"additionalProperties\": false } ```
", "type": "object", "properties": { "order": { + "description": "An ordered list of components to attempt to boot from.", "type": "array", "items": { "$ref": "#/components/schemas/BootOrderEntry" @@ -1908,7 +2087,8 @@ }, "required": [ "order" - ] + ], + "additionalProperties": false }, "BootstoreStatus": { "type": "object", @@ -3112,6 +3292,14 @@ "port": { "description": "Switchport to use for external connectivity", "type": "string" + }, + "tx_eq": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] } }, "required": [ @@ -4446,6 +4634,15 @@ } ] }, + "tx_eq": { + "nullable": true, + "description": "TX-EQ configuration for this port", + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] + }, "uplink_port_fec": { "description": "Port forward error correction type.", "allOf": [ @@ -5157,6 +5354,42 @@ "sync" ] }, + "TxEqConfig": { + "description": "Per-port tx-eq overrides. This can be used to fine-tune the transceiver equalization settings to improve signal integrity.", + "type": "object", + "properties": { + "main": { + "nullable": true, + "description": "Main tap", + "type": "integer", + "format": "int32" + }, + "post1": { + "nullable": true, + "description": "Post-cursor tap1", + "type": "integer", + "format": "int32" + }, + "post2": { + "nullable": true, + "description": "Post-cursor tap2", + "type": "integer", + "format": "int32" + }, + "pre1": { + "nullable": true, + "description": "Pre-cursor tap1", + "type": "integer", + "format": "int32" + }, + "pre2": { + "nullable": true, + "description": "Pre-cursor tap2", + "type": "integer", + "format": "int32" + } + } + }, "TypedUuidForDatasetKind": { "type": "string", "format": "uuid" diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 55db469210..8e20f23d06 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -5991,6 +5991,42 @@ } ] }, + "TxEqConfig": { + "description": "Per-port tx-eq overrides. This can be used to fine-tune the transceiver equalization settings to improve signal integrity.", + "type": "object", + "properties": { + "main": { + "nullable": true, + "description": "Main tap", + "type": "integer", + "format": "int32" + }, + "post1": { + "nullable": true, + "description": "Post-cursor tap1", + "type": "integer", + "format": "int32" + }, + "post2": { + "nullable": true, + "description": "Post-cursor tap2", + "type": "integer", + "format": "int32" + }, + "pre1": { + "nullable": true, + "description": "Pre-cursor tap1", + "type": "integer", + "format": "int32" + }, + "pre2": { + "nullable": true, + "description": "Pre-cursor tap2", + "type": "integer", + "format": "int32" + } + } + }, "TypedUuidForRackInitKind": { "type": "string", "format": "uuid" @@ -6630,6 +6666,15 @@ "$ref": "#/components/schemas/RouteConfig" } }, + "tx_eq": { + "nullable": true, + "default": null, + "allOf": [ + { + "$ref": "#/components/schemas/TxEqConfig" + } + ] + }, "uplink_port_fec": { "$ref": "#/components/schemas/PortFec" }, diff --git a/oximeter/collector/src/lib.rs b/oximeter/collector/src/lib.rs index 6b10cf31cd..68fff4cbf0 100644 --- a/oximeter/collector/src/lib.rs +++ b/oximeter/collector/src/lib.rs @@ -344,11 +344,20 @@ impl Oximeter { )) }; - qorb::pool::Pool::new( + match qorb::pool::Pool::new( nexus_resolver, Arc::new(NexusConnector { log: log.clone() }), qorb::policy::Policy::default(), - ) + ) { + Ok(pool) => { + debug!(log, "registered USDT probes"); + pool + } + Err(err) => { + error!(log, "failed to register USDT probes"); + err.into_inner() + } + } }; let notify_nexus = || async { diff --git a/oximeter/db/schema/replicated/13/up1.sql b/oximeter/db/schema/replicated/13/up1.sql new file mode 100644 index 0000000000..ac13ea6d01 --- /dev/null +++ b/oximeter/db/schema/replicated/13/up1.sql @@ -0,0 +1,4 @@ +ALTER TABLE oximeter.fields_bool_local +ON CLUSTER oximeter_cluster +ALTER COLUMN IF EXISTS +field_value TYPE Bool; diff --git a/oximeter/db/schema/replicated/13/up2.sql b/oximeter/db/schema/replicated/13/up2.sql new file mode 100644 index 0000000000..11f42784c5 --- /dev/null +++ b/oximeter/db/schema/replicated/13/up2.sql @@ -0,0 +1,4 @@ +ALTER TABLE oximeter.measurements_bool_local +ON CLUSTER oximeter_cluster +ALTER COLUMN IF EXISTS +datum TYPE Nullable(Bool); diff --git a/oximeter/db/schema/replicated/13/up3.sql b/oximeter/db/schema/replicated/13/up3.sql new file mode 100644 index 0000000000..e39f7d1c73 --- /dev/null +++ b/oximeter/db/schema/replicated/13/up3.sql @@ -0,0 +1,4 @@ +ALTER TABLE oximeter.fields_bool +ON CLUSTER oximeter_cluster +ALTER COLUMN IF EXISTS +field_value TYPE Bool; diff --git a/oximeter/db/schema/replicated/13/up4.sql b/oximeter/db/schema/replicated/13/up4.sql new file mode 100644 index 0000000000..03aec4da41 --- /dev/null +++ b/oximeter/db/schema/replicated/13/up4.sql @@ -0,0 +1,4 @@ +ALTER TABLE oximeter.measurements_bool +ON CLUSTER oximeter_cluster +ALTER COLUMN IF EXISTS +datum TYPE Nullable(Bool); diff --git a/oximeter/db/schema/replicated/db-init-2.sql b/oximeter/db/schema/replicated/db-init-2.sql index 51e64e20e0..7507041f40 100644 --- a/oximeter/db/schema/replicated/db-init-2.sql +++ b/oximeter/db/schema/replicated/db-init-2.sql @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool_local ON CLUSTER oximeter_ timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Nullable(UInt8) + datum Nullable(Bool) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_bool_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -595,7 +595,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_bool_local ON CLUSTER oximeter_cluste timeseries_name String, timeseries_key UInt64, field_name String, - field_value UInt8, + field_value Bool, last_updated_at DateTime MATERIALIZED now() ) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/fields_bool_local', '{replica}') diff --git a/oximeter/db/schema/single-node/13/up1.sql b/oximeter/db/schema/single-node/13/up1.sql new file mode 100644 index 0000000000..36ffaad12b --- /dev/null +++ b/oximeter/db/schema/single-node/13/up1.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.fields_bool ALTER COLUMN IF EXISTS field_value TYPE Bool; diff --git a/oximeter/db/schema/single-node/13/up2.sql b/oximeter/db/schema/single-node/13/up2.sql new file mode 100644 index 0000000000..b7b2c9b299 --- /dev/null +++ b/oximeter/db/schema/single-node/13/up2.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_bool ALTER COLUMN IF EXISTS datum TYPE Nullable(Bool); diff --git a/oximeter/db/schema/single-node/db-init.sql b/oximeter/db/schema/single-node/db-init.sql index 184951feeb..d505a21414 100644 --- a/oximeter/db/schema/single-node/db-init.sql +++ b/oximeter/db/schema/single-node/db-init.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Nullable(UInt8) + datum Nullable(Bool) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -518,7 +518,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_bool timeseries_name String, timeseries_key UInt64, field_name String, - field_value UInt8, + field_value Bool, last_updated_at DateTime MATERIALIZED now() ) ENGINE = ReplacingMergeTree() diff --git a/oximeter/db/src/bin/oxdb/main.rs b/oximeter/db/src/bin/oxdb/main.rs index ad5018eee5..e7d49f6707 100644 --- a/oximeter/db/src/bin/oxdb/main.rs +++ b/oximeter/db/src/bin/oxdb/main.rs @@ -311,8 +311,6 @@ async fn query( #[tokio::main] async fn main() -> anyhow::Result<()> { - usdt::register_probes().context("Failed to register USDT probes")?; - let args = OxDb::parse(); let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::FullFormat::new(decorator) diff --git a/oximeter/db/src/client/mod.rs b/oximeter/db/src/client/mod.rs index 39051e70c8..3fdee90858 100644 --- a/oximeter/db/src/client/mod.rs +++ b/oximeter/db/src/client/mod.rs @@ -195,21 +195,39 @@ impl Client { )); let schema = Mutex::new(BTreeMap::new()); let request_timeout = DEFAULT_REQUEST_TIMEOUT; + let pool = match Pool::new( + http_resolver, + Arc::new(ReqwestConnector {}), + qorb::policy::Policy::default(), + ) { + Ok(pool) => { + debug!(log, "registered USDT probes"); + pool + } + Err(err) => { + error!(log, "failed to register USDT probes"); + err.into_inner() + } + }; + let native_pool = match Pool::new( + native_resolver, + Arc::new(native::connection::Connector), + qorb::policy::Policy::default(), + ) { + Ok(pool) => { + debug!(log, "registered USDT probes"); + pool + } + Err(err) => { + error!(log, "failed to register USDT probes"); + err.into_inner() + } + }; Self { _id: id, log, - source: ClientSource::Pool { - pool: DebugIgnore(Pool::new( - http_resolver, - Arc::new(ReqwestConnector {}), - qorb::policy::Policy::default(), - )), - }, - native_pool: DebugIgnore(Pool::new( - native_resolver, - Arc::new(native::connection::Connector), - Default::default(), - )), + source: ClientSource::Pool { pool: DebugIgnore(pool) }, + native_pool: DebugIgnore(native_pool), schema, request_timeout, } @@ -245,15 +263,25 @@ impl Client { let client = reqwest::Client::new(); let url = format!("http://{}", http_address); let schema = Mutex::new(BTreeMap::new()); + let native_pool = match Pool::new( + Box::new(SingleHostResolver::new(native_address)), + Arc::new(native::connection::Connector), + Default::default(), + ) { + Ok(pool) => { + debug!(log, "registered USDT probes"); + pool + } + Err(err) => { + error!(log, "failed to register USDT probes"); + err.into_inner() + } + }; Self { _id: id, log, source: ClientSource::Static(ReqwestClient { url, client }), - native_pool: DebugIgnore(Pool::new( - Box::new(SingleHostResolver::new(native_address)), - Arc::new(native::connection::Connector), - Default::default(), - )), + native_pool: DebugIgnore(native_pool), schema, request_timeout, } @@ -1194,17 +1222,16 @@ impl Client { })?; // Convert the HTTP response into a database response. - let response = handle_db_response(response).await.map_err(|err| { - probes::sql__query__done!(|| (&id)); - err - })?; + let response = + handle_db_response(response).await.inspect_err(|_| { + probes::sql__query__done!(|| (&id)); + })?; // Extract the query summary, measuring resource usage and duration. let summary = QuerySummary::from_headers(start.elapsed(), response.headers()) - .map_err(|err| { + .inspect_err(|_| { probes::sql__query__done!(|| (&id)); - err })?; // Extract the actual text of the response. @@ -1798,7 +1825,7 @@ mod tests { .ping() .await .expect_err("Should fail to ping non-existent server"); - let Error::Connection(qorb::pool::Error::TimedOut) = &e else { + let Error::Connection(_) = &e else { panic!("Expected connection error, found {e:?}"); }; logctx.cleanup_successful(); @@ -3022,7 +3049,7 @@ mod tests { client: &Client, ) -> Result<(), Error> { let field = FieldValue::Bool(true); - let as_json = serde_json::Value::from(1_u64); + let as_json = serde_json::Value::from(true); test_recall_field_value_impl(field, as_json, client).await?; Ok(()) } @@ -3729,7 +3756,6 @@ mod tests { _: &ClickHouseDeployment, client: Client, ) { - usdt::register_probes().unwrap(); let samples = [oximeter_test_utils::make_sample()]; client.insert_samples(&samples).await.unwrap(); @@ -3789,7 +3815,6 @@ mod tests { client: Client, ) { use strum::IntoEnumIterator; - usdt::register_probes().unwrap(); // Attempt to select all schema with each datum type. for ty in oximeter::DatumType::iter() { let sql = format!( @@ -3830,7 +3855,6 @@ mod tests { db: &ClickHouseDeployment, client: Client, ) { - usdt::register_probes().unwrap(); let samples = [oximeter_test_utils::make_sample()]; // We're using the components of the `insert_samples()` method here, @@ -4436,7 +4460,6 @@ mod tests { #[tokio::test] async fn test_select_all_field_types() { use strum::IntoEnumIterator; - usdt::register_probes().unwrap(); let logctx = test_setup_log("test_select_all_field_types"); let log = &logctx.log; @@ -4855,7 +4878,6 @@ mod tests { native_address: SocketAddr, replicated: bool, ) { - usdt::register_probes().unwrap(); let client = Client::new(http_address, native_address, &log); const STARTING_VERSION: u64 = 1; diff --git a/oximeter/db/src/client/oxql.rs b/oximeter/db/src/client/oxql.rs index d9f3295375..184e70cd44 100644 --- a/oximeter/db/src/client/oxql.rs +++ b/oximeter/db/src/client/oxql.rs @@ -1385,7 +1385,6 @@ mod tests { // fetching different sets of fields at different times. #[tokio::test] async fn test_get_entire_timeseries_and_part_of_another() { - usdt::register_probes().unwrap(); let ctx = setup_oxql_test("test_get_entire_timeseries_and_part_of_another") .await; diff --git a/oximeter/db/src/configs/replica_config.xml b/oximeter/db/src/configs/replica_config.xml index bf424185b6..7d27b51ab4 100644 --- a/oximeter/db/src/configs/replica_config.xml +++ b/oximeter/db/src/configs/replica_config.xml @@ -484,4 +484,9 @@ false false + + + + 1.0 + diff --git a/oximeter/db/src/model/mod.rs b/oximeter/db/src/model/mod.rs index eba2333ae5..93b857f34f 100644 --- a/oximeter/db/src/model/mod.rs +++ b/oximeter/db/src/model/mod.rs @@ -50,7 +50,7 @@ pub mod to_block; /// - [`crate::Client::initialize_db_with_version`] /// - [`crate::Client::ensure_schema`] /// - The `clickhouse-schema-updater` binary in this crate -pub const OXIMETER_VERSION: u64 = 12; +pub const OXIMETER_VERSION: u64 = 13; // Wrapper type to represent a boolean in the database. // @@ -401,7 +401,7 @@ macro_rules! declare_field_row { } } -declare_field_row! {BoolFieldRow, DbBool, "bool"} +declare_field_row! {BoolFieldRow, bool, "bool"} declare_field_row! {I8FieldRow, i8, "i8"} declare_field_row! {U8FieldRow, u8, "u8"} declare_field_row! {I16FieldRow, i16, "i16"} @@ -639,7 +639,7 @@ fn unroll_from_source(sample: &Sample) -> BTreeMap> { timeseries_name, timeseries_key, field_name, - field_value: DbBool::from(*inner), + field_value: *inner, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } @@ -1477,7 +1477,7 @@ trait FromDbScalar { const DATUM_TYPE: DatumType; } -impl FromDbScalar for DbBool { +impl FromDbScalar for bool { const DATUM_TYPE: DatumType = DatumType::Bool; } @@ -1642,7 +1642,7 @@ pub(crate) fn parse_measurement_from_row( ) -> (TimeseriesKey, Measurement) { match datum_type { DatumType::Bool => { - parse_timeseries_scalar_gauge_measurement::(line) + parse_timeseries_scalar_gauge_measurement::(line) } DatumType::I8 => parse_timeseries_scalar_gauge_measurement::(line), DatumType::U8 => parse_timeseries_scalar_gauge_measurement::(line), @@ -1766,11 +1766,11 @@ pub(crate) fn parse_field_select_row( // Parse the field value as the expected type let value = match expected_field.field_type { FieldType::Bool => { - FieldValue::Bool(bool::from(DbBool::from( + FieldValue::Bool( actual_field_value - .as_u64() - .expect("Expected a u64 for a boolean field from the database") - ))) + .as_bool() + .expect("Expected a boolean field from the database") + ) } FieldType::I8 => { let wide = actual_field_value @@ -2080,7 +2080,7 @@ mod tests { assert_eq!(measurement.datum(), datum); } - let line = r#"{"timeseries_key": 12, "timestamp": "2021-01-01 00:00:00.123456789", "datum": 1 }"#; + let line = r#"{"timeseries_key": 12, "timestamp": "2021-01-01 00:00:00.123456789", "datum": true }"#; let datum = Datum::from(true); run_test(line, &datum, timestamp); diff --git a/oximeter/db/src/shells/native.rs b/oximeter/db/src/shells/native.rs index f513435275..ae075aa02f 100644 --- a/oximeter/db/src/shells/native.rs +++ b/oximeter/db/src/shells/native.rs @@ -16,7 +16,6 @@ use tabled::{builder::Builder, settings::Style}; /// Run the native SQL shell. pub async fn shell(addr: IpAddr, port: u16) -> anyhow::Result<()> { - usdt::register_probes()?; let addr = SocketAddr::new(addr, port); let mut conn = native::Connection::new(addr) .await diff --git a/oximeter/db/tests/integration_test.rs b/oximeter/db/tests/integration_test.rs index 3a1649959e..b34c962881 100644 --- a/oximeter/db/tests/integration_test.rs +++ b/oximeter/db/tests/integration_test.rs @@ -123,7 +123,7 @@ async fn test_schemas_disjoint() -> anyhow::Result<()> { /// doesn't make much sense in an integration test. #[tokio::test] async fn test_cluster() -> anyhow::Result<()> { - usdt::register_probes().unwrap(); + usdt::register_probes().expect("Failed to register USDT probes"); let request_timeout = Duration::from_secs(15); let start = tokio::time::Instant::now(); let logctx = test_setup_log("test_cluster"); diff --git a/oximeter/types/src/histogram.rs b/oximeter/types/src/histogram.rs index 2507f2f5c6..a1ec458b73 100644 --- a/oximeter/types/src/histogram.rs +++ b/oximeter/types/src/histogram.rs @@ -1061,7 +1061,7 @@ where let lo = base.pow(lo as _); let hi = base.pow(hi as _); let distance = hi - lo; - distance.is_multiple_of(&count) + Integer::is_multiple_of(&distance, &count) }) } diff --git a/package-manifest.toml b/package-manifest.toml index 2b5eecc2ec..fffa89640d 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -578,10 +578,10 @@ only_for_targets.image = "standard" # 3. Use source.type = "manual" instead of "prebuilt" source.type = "prebuilt" source.repo = "crucible" -source.commit = "2b88ab88461fb06aaf2aab11c5e381a3cad25eac" +source.commit = "b7b9d5660b28ca5e865242b2bdecd032c0852d40" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible.sha256.txt -source.sha256 = "0927e7211f645ef8c3f715c7b6a8be81a0f4b6250a3ff352348ae701e89713f5" +source.sha256 = "44e623730765f8fc0b702d107939552514530a33b306ca5e8bc8276ff0aaf79a" output.type = "zone" output.intermediate_only = true @@ -590,10 +590,10 @@ service_name = "crucible_pantry_prebuilt" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "2b88ab88461fb06aaf2aab11c5e381a3cad25eac" +source.commit = "b7b9d5660b28ca5e865242b2bdecd032c0852d40" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-pantry.sha256.txt -source.sha256 = "c273dd273cb09cbd8007925b41fcc6df9807dd93e395347b36ce8306a8dc93e4" +source.sha256 = "bc0a41d349646ec2111bff346db2c300001d646a99f33b05b39b78188e34ae41" output.type = "zone" output.intermediate_only = true @@ -607,10 +607,10 @@ service_name = "crucible_dtrace" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "2b88ab88461fb06aaf2aab11c5e381a3cad25eac" +source.commit = "b7b9d5660b28ca5e865242b2bdecd032c0852d40" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-dtrace.sha256.txt -source.sha256 = "972a713eb02bc1aeaa7b16db24fef7f82afcfc4546abe998792c85f4a48269c0" +source.sha256 = "64e37f7a062f7c8941fac3b95a81d98475e5c02ff01111554b0ddb7fc232f40f" output.type = "tarball" # Refer to @@ -621,10 +621,10 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "11371b0f3743f8df5b047dc0edc2699f4bdf3927" +source.commit = "86101eaf80b55e7f405b5cafe9b0de0e9f331656" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "07383cbad45bc032de1b65d3553839751fde96342cc76249ca4a45b89872aae9" +source.sha256 = "8dd411d6f2db23f93c2340cce11aa194da8dcb8cfd20081a614a5722ffbfe255" output.type = "zone" [package.mg-ddm-gz] @@ -719,8 +719,8 @@ only_for_targets.image = "standard" # the other `source.*` keys. source.type = "prebuilt" source.repo = "dendrite" -source.commit = "f3810e7bc1f0d746b5e95b3aaff32e52b02dfdfa" -source.sha256 = "c1506f6f818327523e6ff3102432a2038d319338b883235664b34f9132ff676a" +source.commit = "4067d742d832fa434217b95e4b149048d01ef54e" +source.sha256 = "5e9ccc42e5ac31f4be24025d2afd5978aef33d618f3cb7caa260eff73b7e6a79" output.type = "zone" output.intermediate_only = true @@ -746,8 +746,8 @@ only_for_targets.image = "standard" # the other `source.*` keys. source.type = "prebuilt" source.repo = "dendrite" -source.commit = "f3810e7bc1f0d746b5e95b3aaff32e52b02dfdfa" -source.sha256 = "061d40085e733e60d7c53ebfd2a4cf64f54a856e7eb5fd4b82ac65ec6a5b847b" +source.commit = "4067d742d832fa434217b95e4b149048d01ef54e" +source.sha256 = "9d3156b7895126b9df5460dd0c34668738a7f2d5894a4be0229644820e732895" output.type = "zone" output.intermediate_only = true @@ -766,8 +766,8 @@ only_for_targets.image = "standard" # the other `source.*` keys. source.type = "prebuilt" source.repo = "dendrite" -source.commit = "f3810e7bc1f0d746b5e95b3aaff32e52b02dfdfa" -source.sha256 = "c6cb4c077f0ddfc78ab06e07316d1312657f95526ced60c2b8e7baf1c73ae24a" +source.commit = "4067d742d832fa434217b95e4b149048d01ef54e" +source.sha256 = "4eff4f00201ab8373510644693d066dbec2497142d48964be9844f0b30c147e8" output.type = "zone" output.intermediate_only = true diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index cd88345d0a..c828c90432 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -1069,22 +1069,20 @@ async fn main() -> Result<()> { let get_config = || -> Result { let target_path = args.artifact_dir.join("target").join(&args.target); let raw_target = - std::fs::read_to_string(&target_path).map_err(|e| { + std::fs::read_to_string(&target_path).inspect_err(|_| { eprintln!( "Failed to read build target: {}\n{}", target_path, target_help_str() ); - e })?; let target: Target = KnownTarget::from_str(&raw_target) - .map_err(|e| { + .inspect_err(|_| { eprintln!( "Failed to parse {} as target\n{}", target_path, target_help_str() ); - e })? .into(); debug!(log, "target[{}]: {:?}", args.target, target); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index e3dc6ba131..aed265e0e6 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] # We choose a specific toolchain (rather than "stable") for repeatability. The # intent is to keep this up-to-date with recently-released stable Rust. -channel = "1.80.1" +channel = "1.82.0" profile = "default" diff --git a/schema/crdb/README.adoc b/schema/crdb/README.adoc index 3567821ea6..a9c437bc9a 100644 --- a/schema/crdb/README.adoc +++ b/schema/crdb/README.adoc @@ -168,6 +168,34 @@ If you cannot write the data migration in SQL, you would need to figure out a different way to backfill the data before you can apply the step that adds the `NOT NULL` constraint. This is likely a substantial project +==== Changing enum variants + +Adding a new variant to an enum is straightforward: `ALTER TYPE your_type ADD VALUE IF NOT EXISTS your_new_value AFTER some_existing_value` +(or `... BEFORE some_existing_value`); for an example, see the +link:https://github.com/oxidecomputer/omicron/tree/main/schema/crdb/add-management-gateway-producer-kind[`add-management-gateway-producer-kind`] migration. + +Removing or renaming variants is more burdensome. `ALTER TYPE DROP VALUE ...` +and `ALTER TYPE RENAME VALUE ...` both exist, but they do not have clauses to +support idempotent operation, making them unsuitable for migrations. Instead, +you can use the following sequence of migration steps: + +. Create a new temporary enum with the new variants, and a different name as the old type. +. Create a new temporary column with the temporary enum type. (Adding a column supports `IF NOT EXISTS`). +. Set the values of the temporary column based on the value of the old column. +. Drop the old column. +. Drop the old type. +. Create a new enum with the new variants, and the same name as the original enum type (which we can now do, as the old type has been dropped). +. Create a new column with the same name as the original column, and the new type --- again, we can do this now as the original column has been dropped. +. Set the values of the new column based on the temporary column. +. Drop the temporary column. +. Drop the temporary type. + +For an example, see the +link:https://github.com/oxidecomputer/omicron/tree/main/schema/crdb/auto-restart-policy-v2[`auto-restart-policy-v2`] migration (whose README is the +source of this list!). The steps can be simplified some if the enum itself is +being renamed, in which case you may not need the temporary enum; see the +link:https://github.com/oxidecomputer/omicron/tree/main/schema/crdb/separate-instance-and-vmm-states[`separate-instance-and-vmm-states`] migration for an example. + ==== Renaming columns Idempotently renaming existing columns is unfortunately not possible in our diff --git a/schema/crdb/add-tx-eq/up1.sql b/schema/crdb/add-tx-eq/up1.sql new file mode 100644 index 0000000000..07be4dd657 --- /dev/null +++ b/schema/crdb/add-tx-eq/up1.sql @@ -0,0 +1,4 @@ +/* + * Add a pointer to this link's transceiver equalization config settings. + */ +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS tx_eq_config_id UUID; diff --git a/schema/crdb/add-tx-eq/up2.sql b/schema/crdb/add-tx-eq/up2.sql new file mode 100644 index 0000000000..76c8083eaa --- /dev/null +++ b/schema/crdb/add-tx-eq/up2.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS omicron.public.tx_eq_config ( + id UUID PRIMARY KEY, + pre1 INT4, + pre2 INT4, + main INT4, + post2 INT4, + post1 INT4 +); diff --git a/schema/crdb/blueprint-dataset/up01.sql b/schema/crdb/blueprint-dataset/up01.sql new file mode 100644 index 0000000000..cfdde5bacd --- /dev/null +++ b/schema/crdb/blueprint-dataset/up01.sql @@ -0,0 +1,4 @@ +ALTER TABLE omicron.public.dataset + ADD COLUMN IF NOT EXISTS quota INT8, + ADD COLUMN IF NOT EXISTS reservation INT8, + ADD COLUMN IF NOT EXISTS compression TEXT diff --git a/schema/crdb/blueprint-dataset/up02.sql b/schema/crdb/blueprint-dataset/up02.sql new file mode 100644 index 0000000000..a1a0dc7cb7 --- /dev/null +++ b/schema/crdb/blueprint-dataset/up02.sql @@ -0,0 +1,4 @@ +CREATE TYPE IF NOT EXISTS omicron.public.bp_dataset_disposition AS ENUM ( + 'in_service', + 'expunged' +) diff --git a/schema/crdb/blueprint-dataset/up03.sql b/schema/crdb/blueprint-dataset/up03.sql new file mode 100644 index 0000000000..2ce95db275 --- /dev/null +++ b/schema/crdb/blueprint-dataset/up03.sql @@ -0,0 +1,9 @@ +-- description of a collection of omicron datasets stored in a blueprint +CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_omicron_datasets ( + -- foreign key into the `blueprint` table + blueprint_id UUID NOT NULL, + sled_id UUID NOT NULL, + generation INT8 NOT NULL, + + PRIMARY KEY (blueprint_id, sled_id) +) diff --git a/schema/crdb/blueprint-dataset/up04.sql b/schema/crdb/blueprint-dataset/up04.sql new file mode 100644 index 0000000000..1d21ff5db1 --- /dev/null +++ b/schema/crdb/blueprint-dataset/up04.sql @@ -0,0 +1,35 @@ +-- description of an omicron dataset specified in a blueprint. +CREATE TABLE IF NOT EXISTS omicron.public.bp_omicron_dataset ( + -- foreign key into the `blueprint` table + blueprint_id UUID NOT NULL, + sled_id UUID NOT NULL, + id UUID NOT NULL, + + -- Dataset disposition + disposition omicron.public.bp_dataset_disposition NOT NULL, + + pool_id UUID NOT NULL, + kind omicron.public.dataset_kind NOT NULL, + -- Only valid if kind = zone + zone_name TEXT, + + -- Only valid if kind = crucible + ip INET, + port INT4 CHECK (port BETWEEN 0 AND 65535), + + quota INT8, + reservation INT8, + compression TEXT NOT NULL, + + CONSTRAINT zone_name_for_zone_kind CHECK ( + (kind != 'zone') OR + (kind = 'zone' AND zone_name IS NOT NULL) + ), + + CONSTRAINT ip_and_port_set_for_crucible CHECK ( + (kind != 'crucible') OR + (kind = 'crucible' AND ip IS NOT NULL and port IS NOT NULL) + ), + + PRIMARY KEY (blueprint_id, id) +) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index e4c82c77f9..b095c4c89a 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -582,6 +582,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.dataset ( /* Only valid if kind = zone -- the name of this zone */ zone_name TEXT, + quota INT8, + reservation INT8, + compression TEXT, + /* Crucible must make use of 'size_used'; other datasets manage their own storage */ CONSTRAINT size_used_column_set_for_crucible CHECK ( (kind != 'crucible') OR @@ -2843,6 +2847,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_link_config ( speed omicron.public.switch_link_speed, autoneg BOOL NOT NULL DEFAULT false, lldp_link_config_id UUID, + tx_eq_config_id UUID, PRIMARY KEY (port_settings_id, link_name) ); @@ -2861,6 +2866,15 @@ CREATE TABLE IF NOT EXISTS omicron.public.lldp_link_config ( time_deleted TIMESTAMPTZ ); +CREATE TABLE IF NOT EXISTS omicron.public.tx_eq_config ( + id UUID PRIMARY KEY, + pre1 INT4, + pre2 INT4, + main INT4, + post2 INT4, + post1 INT4 +); + CREATE TYPE IF NOT EXISTS omicron.public.switch_interface_kind AS ENUM ( 'primary', 'vlan', @@ -3600,6 +3614,11 @@ CREATE TYPE IF NOT EXISTS omicron.public.bp_zone_disposition AS ENUM ( 'expunged' ); +CREATE TYPE IF NOT EXISTS omicron.public.bp_dataset_disposition AS ENUM ( + 'in_service', + 'expunged' +); + -- list of all blueprints CREATE TABLE IF NOT EXISTS omicron.public.blueprint ( id UUID PRIMARY KEY, @@ -3701,6 +3720,52 @@ CREATE TABLE IF NOT EXISTS omicron.public.bp_omicron_physical_disk ( PRIMARY KEY (blueprint_id, id) ); +-- description of a collection of omicron datasets stored in a blueprint +CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_omicron_datasets ( + -- foreign key into the `blueprint` table + blueprint_id UUID NOT NULL, + sled_id UUID NOT NULL, + generation INT8 NOT NULL, + + PRIMARY KEY (blueprint_id, sled_id) +); + +-- description of an omicron dataset specified in a blueprint. +CREATE TABLE IF NOT EXISTS omicron.public.bp_omicron_dataset ( + -- foreign key into the `blueprint` table + blueprint_id UUID NOT NULL, + sled_id UUID NOT NULL, + id UUID NOT NULL, + + -- Dataset disposition + disposition omicron.public.bp_dataset_disposition NOT NULL, + + pool_id UUID NOT NULL, + kind omicron.public.dataset_kind NOT NULL, + -- Only valid if kind = zone + zone_name TEXT, + + -- Only valid if kind = crucible + ip INET, + port INT4 CHECK (port BETWEEN 0 AND 65535), + + quota INT8, + reservation INT8, + compression TEXT NOT NULL, + + CONSTRAINT zone_name_for_zone_kind CHECK ( + (kind != 'zone') OR + (kind = 'zone' AND zone_name IS NOT NULL) + ), + + CONSTRAINT ip_and_port_set_for_crucible CHECK ( + (kind != 'crucible') OR + (kind = 'crucible' AND ip IS NOT NULL and port IS NOT NULL) + ), + + PRIMARY KEY (blueprint_id, id) +); + -- see inv_sled_omicron_zones, which is identical except it references a -- collection whereas this table references a blueprint CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_omicron_zones ( @@ -4536,7 +4601,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '111.0.0', NULL) + (TRUE, NOW(), NOW(), '113.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/rss-service-plan-v5.json b/schema/rss-service-plan-v5.json new file mode 100644 index 0000000000..132b27c10e --- /dev/null +++ b/schema/rss-service-plan-v5.json @@ -0,0 +1,1197 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Plan", + "type": "object", + "required": [ + "dns_config", + "services" + ], + "properties": { + "dns_config": { + "$ref": "#/definitions/DnsConfigParams" + }, + "services": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/SledConfig" + } + } + }, + "definitions": { + "BlueprintZoneConfig": { + "description": "Describes one Omicron-managed zone in a blueprint.\n\nPart of [`BlueprintZonesConfig`].", + "type": "object", + "required": [ + "disposition", + "id", + "zone_type" + ], + "properties": { + "disposition": { + "description": "The disposition (desired state) of this zone recorded in the blueprint.", + "allOf": [ + { + "$ref": "#/definitions/BlueprintZoneDisposition" + } + ] + }, + "filesystem_pool": { + "description": "zpool used for the zone's (transient) root filesystem", + "anyOf": [ + { + "$ref": "#/definitions/ZpoolName" + }, + { + "type": "null" + } + ] + }, + "id": { + "$ref": "#/definitions/TypedUuidForOmicronZoneKind" + }, + "zone_type": { + "$ref": "#/definitions/BlueprintZoneType" + } + } + }, + "BlueprintZoneDisposition": { + "description": "The desired state of an Omicron-managed zone in a blueprint.\n\nPart of [`BlueprintZoneConfig`].", + "oneOf": [ + { + "description": "The zone is in-service.", + "type": "string", + "enum": [ + "in_service" + ] + }, + { + "description": "The zone is not in service.", + "type": "string", + "enum": [ + "quiesced" + ] + }, + { + "description": "The zone is permanently gone.", + "type": "string", + "enum": [ + "expunged" + ] + } + ] + }, + "BlueprintZoneType": { + "oneOf": [ + { + "type": "object", + "required": [ + "address", + "dns_servers", + "external_ip", + "nic", + "ntp_servers", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dns_servers": { + "type": "array", + "items": { + "type": "string", + "format": "ip" + } + }, + "domain": { + "type": [ + "string", + "null" + ] + }, + "external_ip": { + "$ref": "#/definitions/OmicronZoneExternalSnatIp" + }, + "nic": { + "description": "The service vNIC providing outbound connectivity using OPTE.", + "allOf": [ + { + "$ref": "#/definitions/NetworkInterface" + } + ] + }, + "ntp_servers": { + "type": "array", + "items": { + "type": "string" + } + }, + "type": { + "type": "string", + "enum": [ + "boundary_ntp" + ] + } + } + }, + { + "description": "Used in single-node clickhouse setups", + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "clickhouse" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "clickhouse_keeper" + ] + } + } + }, + { + "description": "Used in replicated clickhouse setups", + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "clickhouse_server" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "cockroach_db" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "dataset", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "type": { + "type": "string", + "enum": [ + "crucible" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "crucible_pantry" + ] + } + } + }, + { + "type": "object", + "required": [ + "dataset", + "dns_address", + "http_address", + "nic", + "type" + ], + "properties": { + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "dns_address": { + "description": "The address at which the external DNS server is reachable.", + "allOf": [ + { + "$ref": "#/definitions/OmicronZoneExternalFloatingAddr" + } + ] + }, + "http_address": { + "description": "The address at which the external DNS server API is reachable.", + "type": "string" + }, + "nic": { + "description": "The service vNIC providing external connectivity using OPTE.", + "allOf": [ + { + "$ref": "#/definitions/NetworkInterface" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "external_dns" + ] + } + } + }, + { + "type": "object", + "required": [ + "dataset", + "dns_address", + "gz_address", + "gz_address_index", + "http_address", + "type" + ], + "properties": { + "dataset": { + "$ref": "#/definitions/OmicronZoneDataset" + }, + "dns_address": { + "type": "string" + }, + "gz_address": { + "description": "The addresses in the global zone which should be created\n\nFor the DNS service, which exists outside the sleds's typical subnet - adding an address in the GZ is necessary to allow inter-zone traffic routing.", + "type": "string", + "format": "ipv6" + }, + "gz_address_index": { + "description": "The address is also identified with an auxiliary bit of information to ensure that the created global zone address can have a unique name.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "http_address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "internal_dns" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "internal_ntp" + ] + } + } + }, + { + "type": "object", + "required": [ + "external_dns_servers", + "external_ip", + "external_tls", + "internal_address", + "nic", + "type" + ], + "properties": { + "external_dns_servers": { + "description": "External DNS servers Nexus can use to resolve external hosts.", + "type": "array", + "items": { + "type": "string", + "format": "ip" + } + }, + "external_ip": { + "description": "The address at which the external nexus server is reachable.", + "allOf": [ + { + "$ref": "#/definitions/OmicronZoneExternalFloatingIp" + } + ] + }, + "external_tls": { + "description": "Whether Nexus's external endpoint should use TLS", + "type": "boolean" + }, + "internal_address": { + "description": "The address at which the internal nexus server is reachable.", + "type": "string" + }, + "nic": { + "description": "The service vNIC providing external connectivity using OPTE.", + "allOf": [ + { + "$ref": "#/definitions/NetworkInterface" + } + ] + }, + "type": { + "type": "string", + "enum": [ + "nexus" + ] + } + } + }, + { + "type": "object", + "required": [ + "address", + "type" + ], + "properties": { + "address": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "oximeter" + ] + } + } + } + ] + }, + "ByteCount": { + "description": "Byte count to express memory or storage capacity.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "CompressionAlgorithm": { + "oneOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "on" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "off" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "gzip" + ] + } + } + }, + { + "type": "object", + "required": [ + "level", + "type" + ], + "properties": { + "level": { + "$ref": "#/definitions/GzipLevel" + }, + "type": { + "type": "string", + "enum": [ + "gzip_n" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "lz4" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "lzjb" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "zle" + ] + } + } + } + ] + }, + "DatasetConfig": { + "description": "Configuration information necessary to request a single dataset", + "type": "object", + "required": [ + "compression", + "id", + "name" + ], + "properties": { + "compression": { + "description": "The compression mode to be used by the dataset", + "allOf": [ + { + "$ref": "#/definitions/CompressionAlgorithm" + } + ] + }, + "id": { + "description": "The UUID of the dataset being requested", + "allOf": [ + { + "$ref": "#/definitions/TypedUuidForDatasetKind" + } + ] + }, + "name": { + "description": "The dataset's name", + "allOf": [ + { + "$ref": "#/definitions/DatasetName" + } + ] + }, + "quota": { + "description": "The upper bound on the amount of storage used by this dataset", + "anyOf": [ + { + "$ref": "#/definitions/ByteCount" + }, + { + "type": "null" + } + ] + }, + "reservation": { + "description": "The lower bound on the amount of storage usable by this dataset", + "anyOf": [ + { + "$ref": "#/definitions/ByteCount" + }, + { + "type": "null" + } + ] + } + } + }, + "DatasetKind": { + "description": "The kind of dataset. See the `DatasetKind` enum in omicron-common for possible values.", + "type": "string" + }, + "DatasetName": { + "type": "object", + "required": [ + "kind", + "pool_name" + ], + "properties": { + "kind": { + "$ref": "#/definitions/DatasetKind" + }, + "pool_name": { + "$ref": "#/definitions/ZpoolName" + } + } + }, + "DatasetsConfig": { + "type": "object", + "required": [ + "datasets", + "generation" + ], + "properties": { + "datasets": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/DatasetConfig" + } + }, + "generation": { + "description": "generation number of this configuration\n\nThis generation number is owned by the control plane (i.e., RSS or Nexus, depending on whether RSS-to-Nexus handoff has happened). It should not be bumped within Sled Agent.\n\nSled Agent rejects attempts to set the configuration to a generation older than the one it's currently running.\n\nNote that \"Generation::new()\", AKA, the first generation number, is reserved for \"no datasets\". This is the default configuration for a sled before any requests have been made.", + "allOf": [ + { + "$ref": "#/definitions/Generation" + } + ] + } + } + }, + "DiskIdentity": { + "description": "Uniquely identifies a disk.", + "type": "object", + "required": [ + "model", + "serial", + "vendor" + ], + "properties": { + "model": { + "type": "string" + }, + "serial": { + "type": "string" + }, + "vendor": { + "type": "string" + } + } + }, + "DnsConfigParams": { + "type": "object", + "required": [ + "generation", + "time_created", + "zones" + ], + "properties": { + "generation": { + "$ref": "#/definitions/Generation" + }, + "time_created": { + "type": "string", + "format": "date-time" + }, + "zones": { + "type": "array", + "items": { + "$ref": "#/definitions/DnsConfigZone" + } + } + } + }, + "DnsConfigZone": { + "type": "object", + "required": [ + "records", + "zone_name" + ], + "properties": { + "records": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "$ref": "#/definitions/DnsRecord" + } + } + }, + "zone_name": { + "type": "string" + } + } + }, + "DnsRecord": { + "oneOf": [ + { + "type": "object", + "required": [ + "data", + "type" + ], + "properties": { + "data": { + "type": "string", + "format": "ipv4" + }, + "type": { + "type": "string", + "enum": [ + "A" + ] + } + } + }, + { + "type": "object", + "required": [ + "data", + "type" + ], + "properties": { + "data": { + "type": "string", + "format": "ipv6" + }, + "type": { + "type": "string", + "enum": [ + "AAAA" + ] + } + } + }, + { + "type": "object", + "required": [ + "data", + "type" + ], + "properties": { + "data": { + "$ref": "#/definitions/Srv" + }, + "type": { + "type": "string", + "enum": [ + "SRV" + ] + } + } + } + ] + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "GzipLevel": { + "type": "integer", + "format": "uint8", + "minimum": 0.0 + }, + "IpNet": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/definitions/Ipv4Net" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/definitions/Ipv6Net" + } + ] + } + ], + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::IpNet", + "version": "0.1.0" + } + }, + "Ipv4Net": { + "title": "An IPv4 subnet", + "description": "An IPv4 subnet, including prefix and prefix length", + "examples": [ + "192.168.1.0/24" + ], + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv4Net", + "version": "0.1.0" + } + }, + "Ipv6Net": { + "title": "An IPv6 subnet", + "description": "An IPv6 subnet, including prefix and subnet mask", + "examples": [ + "fd12:3456::/64" + ], + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$", + "x-rust-type": { + "crate": "oxnet", + "path": "oxnet::Ipv6Net", + "version": "0.1.0" + } + }, + "MacAddr": { + "title": "A MAC address", + "description": "A Media Access Control address, in EUI-48 format", + "examples": [ + "ff:ff:ff:ff:ff:ff" + ], + "type": "string", + "maxLength": 17, + "minLength": 5, + "pattern": "^([0-9a-fA-F]{0,2}:){5}[0-9a-fA-F]{0,2}$" + }, + "Name": { + "title": "A name unique within the parent collection", + "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID, but they may contain a UUID. They can be at most 63 characters long.", + "type": "string", + "maxLength": 63, + "minLength": 1, + "pattern": "^(?![0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$)^[a-z]([a-zA-Z0-9-]*[a-zA-Z0-9]+)?$" + }, + "NetworkInterface": { + "description": "Information required to construct a virtual network interface", + "type": "object", + "required": [ + "id", + "ip", + "kind", + "mac", + "name", + "primary", + "slot", + "subnet", + "vni" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "ip": { + "type": "string", + "format": "ip" + }, + "kind": { + "$ref": "#/definitions/NetworkInterfaceKind" + }, + "mac": { + "$ref": "#/definitions/MacAddr" + }, + "name": { + "$ref": "#/definitions/Name" + }, + "primary": { + "type": "boolean" + }, + "slot": { + "type": "integer", + "format": "uint8", + "minimum": 0.0 + }, + "subnet": { + "$ref": "#/definitions/IpNet" + }, + "transit_ips": { + "default": [], + "type": "array", + "items": { + "$ref": "#/definitions/IpNet" + } + }, + "vni": { + "$ref": "#/definitions/Vni" + } + } + }, + "NetworkInterfaceKind": { + "description": "The type of network interface", + "oneOf": [ + { + "description": "A vNIC attached to a guest instance", + "type": "object", + "required": [ + "id", + "type" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "type": { + "type": "string", + "enum": [ + "instance" + ] + } + } + }, + { + "description": "A vNIC associated with an internal service", + "type": "object", + "required": [ + "id", + "type" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "type": { + "type": "string", + "enum": [ + "service" + ] + } + } + }, + { + "description": "A vNIC associated with a probe", + "type": "object", + "required": [ + "id", + "type" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "type": { + "type": "string", + "enum": [ + "probe" + ] + } + } + } + ] + }, + "OmicronPhysicalDiskConfig": { + "type": "object", + "required": [ + "id", + "identity", + "pool_id" + ], + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "identity": { + "$ref": "#/definitions/DiskIdentity" + }, + "pool_id": { + "$ref": "#/definitions/TypedUuidForZpoolKind" + } + } + }, + "OmicronPhysicalDisksConfig": { + "type": "object", + "required": [ + "disks", + "generation" + ], + "properties": { + "disks": { + "type": "array", + "items": { + "$ref": "#/definitions/OmicronPhysicalDiskConfig" + } + }, + "generation": { + "description": "generation number of this configuration\n\nThis generation number is owned by the control plane (i.e., RSS or Nexus, depending on whether RSS-to-Nexus handoff has happened). It should not be bumped within Sled Agent.\n\nSled Agent rejects attempts to set the configuration to a generation older than the one it's currently running.", + "allOf": [ + { + "$ref": "#/definitions/Generation" + } + ] + } + } + }, + "OmicronZoneDataset": { + "description": "Describes a persistent ZFS dataset associated with an Omicron zone", + "type": "object", + "required": [ + "pool_name" + ], + "properties": { + "pool_name": { + "$ref": "#/definitions/ZpoolName" + } + } + }, + "OmicronZoneExternalFloatingAddr": { + "description": "Floating external address with port allocated to an Omicron-managed zone.", + "type": "object", + "required": [ + "addr", + "id" + ], + "properties": { + "addr": { + "type": "string" + }, + "id": { + "$ref": "#/definitions/TypedUuidForExternalIpKind" + } + } + }, + "OmicronZoneExternalFloatingIp": { + "description": "Floating external IP allocated to an Omicron-managed zone.\n\nThis is a slimmer `nexus_db_model::ExternalIp` that only stores the fields necessary for blueprint planning, and requires that the zone have a single IP.", + "type": "object", + "required": [ + "id", + "ip" + ], + "properties": { + "id": { + "$ref": "#/definitions/TypedUuidForExternalIpKind" + }, + "ip": { + "type": "string", + "format": "ip" + } + } + }, + "OmicronZoneExternalSnatIp": { + "description": "SNAT (outbound) external IP allocated to an Omicron-managed zone.\n\nThis is a slimmer `nexus_db_model::ExternalIp` that only stores the fields necessary for blueprint planning, and requires that the zone have a single IP.", + "type": "object", + "required": [ + "id", + "snat_cfg" + ], + "properties": { + "id": { + "$ref": "#/definitions/TypedUuidForExternalIpKind" + }, + "snat_cfg": { + "$ref": "#/definitions/SourceNatConfig" + } + } + }, + "SledConfig": { + "type": "object", + "required": [ + "datasets", + "disks", + "zones" + ], + "properties": { + "datasets": { + "description": "Datasets configured for this sled", + "allOf": [ + { + "$ref": "#/definitions/DatasetsConfig" + } + ] + }, + "disks": { + "description": "Control plane disks configured for this sled", + "allOf": [ + { + "$ref": "#/definitions/OmicronPhysicalDisksConfig" + } + ] + }, + "zones": { + "description": "zones configured for this sled", + "type": "array", + "items": { + "$ref": "#/definitions/BlueprintZoneConfig" + } + } + } + }, + "SourceNatConfig": { + "description": "An IP address and port range used for source NAT, i.e., making outbound network connections from guests or services.", + "type": "object", + "required": [ + "first_port", + "ip", + "last_port" + ], + "properties": { + "first_port": { + "description": "The first port used for source NAT, inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "ip": { + "description": "The external address provided to the instance or service.", + "type": "string", + "format": "ip" + }, + "last_port": { + "description": "The last port used for source NAT, also inclusive.", + "type": "integer", + "format": "uint16", + "minimum": 0.0 + } + } + }, + "Srv": { + "type": "object", + "required": [ + "port", + "prio", + "target", + "weight" + ], + "properties": { + "port": { + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "prio": { + "type": "integer", + "format": "uint16", + "minimum": 0.0 + }, + "target": { + "type": "string" + }, + "weight": { + "type": "integer", + "format": "uint16", + "minimum": 0.0 + } + } + }, + "TypedUuidForDatasetKind": { + "type": "string", + "format": "uuid" + }, + "TypedUuidForExternalIpKind": { + "type": "string", + "format": "uuid" + }, + "TypedUuidForOmicronZoneKind": { + "type": "string", + "format": "uuid" + }, + "TypedUuidForZpoolKind": { + "type": "string", + "format": "uuid" + }, + "Vni": { + "description": "A Geneve Virtual Network Identifier", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "ZpoolName": { + "title": "The name of a Zpool", + "description": "Zpool names are of the format ox{i,p}_. They are either Internal or External, and should be unique", + "type": "string", + "pattern": "^ox[ip]_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + } + } +} \ No newline at end of file diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 7e8513e1e3..56fe35bcde 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -751,6 +751,17 @@ } ] }, + "tx_eq": { + "description": "TX-EQ configuration for this port", + "anyOf": [ + { + "$ref": "#/definitions/TxEqConfig" + }, + { + "type": "null" + } + ] + }, "uplink_port_fec": { "description": "Port forward error correction type.", "allOf": [ @@ -1091,6 +1102,52 @@ } ] }, + "TxEqConfig": { + "description": "Per-port tx-eq overrides. This can be used to fine-tune the transceiver equalization settings to improve signal integrity.", + "type": "object", + "properties": { + "main": { + "description": "Main tap", + "type": [ + "integer", + "null" + ], + "format": "int32" + }, + "post1": { + "description": "Post-cursor tap1", + "type": [ + "integer", + "null" + ], + "format": "int32" + }, + "post2": { + "description": "Post-cursor tap2", + "type": [ + "integer", + "null" + ], + "format": "int32" + }, + "pre1": { + "description": "Pre-cursor tap1", + "type": [ + "integer", + "null" + ], + "format": "int32" + }, + "pre2": { + "description": "Pre-cursor tap2", + "type": [ + "integer", + "null" + ], + "format": "int32" + } + } + }, "TypedUuidForSledKind": { "type": "string", "format": "uuid" diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 360ba7f499..557dcbcb4e 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -64,12 +64,15 @@ propolis_api_types.workspace = true propolis-client.workspace = true propolis-mock-server.workspace = true # Only used by the simulated sled agent rand = { workspace = true, features = ["getrandom"] } +repo-depot-api.workspace = true +repo-depot-client.workspace = true reqwest = { workspace = true, features = ["rustls-tls", "stream"] } schemars = { workspace = true, features = ["chrono", "uuid1"] } semver.workspace = true serde.workspace = true serde_human_bytes.workspace = true serde_json = { workspace = true, features = ["raw_value"] } +sha2.workspace = true sha3.workspace = true sled-agent-api.workspace = true sled-agent-client.workspace = true @@ -105,6 +108,7 @@ opte-ioctl.workspace = true assert_matches.workspace = true expectorate.workspace = true guppy.workspace = true +hex-literal.workspace = true http.workspace = true hyper.workspace = true omicron-test-utils.workspace = true diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index e0d76a857b..b5608602f2 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -2,13 +2,15 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::{collections::BTreeMap, time::Duration}; +use std::collections::BTreeMap; +use std::time::Duration; use camino::Utf8PathBuf; use dropshot::{ - FreeformBody, HttpError, HttpResponseCreated, HttpResponseDeleted, - HttpResponseHeaders, HttpResponseOk, HttpResponseUpdatedNoContent, Path, - Query, RequestContext, StreamingBody, TypedBody, + FreeformBody, HttpError, HttpResponseAccepted, HttpResponseCreated, + HttpResponseDeleted, HttpResponseHeaders, HttpResponseOk, + HttpResponseUpdatedNoContent, Path, Query, RequestContext, StreamingBody, + TypedBody, }; use nexus_sled_agent_shared::inventory::{ Inventory, OmicronZonesConfig, SledRole, @@ -25,6 +27,7 @@ use omicron_common::{ DatasetsConfig, DatasetsManagementResult, DiskVariant, DisksManagementResult, OmicronPhysicalDisksConfig, }, + update::ArtifactHash, }; use omicron_uuid_kinds::{PropolisUuid, ZpoolUuid}; use schemars::JsonSchema; @@ -301,6 +304,43 @@ pub trait SledAgentApi { artifact: TypedBody, ) -> Result; + #[endpoint { + method = GET, + path = "/artifacts" + }] + async fn artifact_list( + rqctx: RequestContext, + ) -> Result>, HttpError>; + + #[endpoint { + method = POST, + path = "/artifacts/{sha256}/copy-from-depot" + }] + async fn artifact_copy_from_depot( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError>; + + #[endpoint { + method = PUT, + path = "/artifacts/{sha256}" + }] + async fn artifact_put( + rqctx: RequestContext, + path_params: Path, + body: StreamingBody, + ) -> Result, HttpError>; + + #[endpoint { + method = DELETE, + path = "/artifacts/{sha256}" + }] + async fn artifact_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + /// Take a snapshot of a disk that is attached to an instance #[endpoint { method = POST, @@ -547,6 +587,30 @@ pub struct DiskPathParam { pub disk_id: Uuid, } +#[derive(Deserialize, JsonSchema)] +pub struct ArtifactPathParam { + pub sha256: ArtifactHash, +} + +#[derive(Deserialize, JsonSchema)] +pub struct ArtifactCopyFromDepotBody { + pub depot_base_url: String, +} + +#[derive(Serialize, JsonSchema)] +pub struct ArtifactCopyFromDepotResponse {} + +#[derive(Debug, Serialize, JsonSchema)] +pub struct ArtifactPutResponse { + /// The number of valid M.2 artifact datasets we found on the sled. There is + /// typically one of these datasets for each functional M.2. + pub datasets: usize, + + /// The number of valid writes to the M.2 artifact datasets. This should be + /// less than or equal to the number of artifact datasets. + pub successful_writes: usize, +} + #[derive(Deserialize, JsonSchema)] pub struct VmmIssueDiskSnapshotRequestPathParam { pub propolis_id: PropolisUuid, diff --git a/sled-agent/repo-depot-api/Cargo.toml b/sled-agent/repo-depot-api/Cargo.toml new file mode 100644 index 0000000000..f9fa60ad8b --- /dev/null +++ b/sled-agent/repo-depot-api/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "repo-depot-api" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[lints] +workspace = true + +[dependencies] +dropshot.workspace = true +omicron-common.workspace = true +omicron-workspace-hack.workspace = true +schemars.workspace = true +serde.workspace = true diff --git a/sled-agent/repo-depot-api/src/lib.rs b/sled-agent/repo-depot-api/src/lib.rs new file mode 100644 index 0000000000..236b9c8e7a --- /dev/null +++ b/sled-agent/repo-depot-api/src/lib.rs @@ -0,0 +1,28 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use dropshot::{FreeformBody, HttpError, HttpResponseOk, Path, RequestContext}; +use omicron_common::update::ArtifactHash; +use schemars::JsonSchema; +use serde::Deserialize; + +#[dropshot::api_description] +pub trait RepoDepotApi { + type Context; + + /// Fetch an artifact from the depot. + #[endpoint { + method = GET, + path = "/artifact/sha256/{sha256}", + }] + async fn artifact_get_by_sha256( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; +} + +#[derive(Clone, Debug, Deserialize, JsonSchema)] +pub struct ArtifactPathParams { + pub sha256: ArtifactHash, +} diff --git a/sled-agent/src/artifact_store.rs b/sled-agent/src/artifact_store.rs new file mode 100644 index 0000000000..fc0dc4a20a --- /dev/null +++ b/sled-agent/src/artifact_store.rs @@ -0,0 +1,910 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Manages TUF artifacts stored on this sled. The implementation is a very +//! basic content-addressed object store. +//! +//! GET operations are handled by the "Repo Depot" API, which is deliberately +//! a separate Dropshot service from the rest of Sled Agent. This is to avoid a +//! circular logical dependency, because we expect Sled Agent to fetch artifacts +//! it does not have from another Repo Depot that does have them (at Nexus's +//! direction). This API's implementation is also part of this module. +//! +//! POST, PUT, and DELETE operations are called by Nexus and handled by the Sled +//! Agent API. + +use std::collections::BTreeMap; +use std::io::ErrorKind; +use std::net::SocketAddrV6; +use std::str::FromStr; +use std::time::Duration; + +use camino::{Utf8Path, Utf8PathBuf}; +use camino_tempfile::{NamedUtf8TempFile, Utf8TempPath}; +use dropshot::{ + Body, ConfigDropshot, FreeformBody, HttpError, HttpResponseOk, + HttpServerStarter, Path, RequestContext, StreamingBody, +}; +use futures::{Stream, TryStreamExt}; +use http::StatusCode; +use omicron_common::address::REPO_DEPOT_PORT; +use omicron_common::disk::{DatasetKind, DatasetsConfig}; +use omicron_common::update::ArtifactHash; +use repo_depot_api::*; +use sha2::{Digest, Sha256}; +use sled_agent_api::ArtifactPutResponse; +use sled_storage::dataset::M2_ARTIFACT_DATASET; +use sled_storage::error::Error as StorageError; +use sled_storage::manager::StorageHandle; +use slog::{error, info, Logger}; +use slog_error_chain::SlogInlineError; +use tokio::fs::{File, OpenOptions}; +use tokio::io::AsyncWriteExt; + +const TEMP_SUBDIR: &str = "tmp"; + +/// Content-addressable local storage for software artifacts. +/// +/// Storage for artifacts is backed by datasets that are explicitly designated +/// for this purpose. The `T: DatasetsManager` parameter, which varies between +/// the real sled agent, the simulated sled agent, and unit tests, specifies +/// exactly which datasets are available for artifact storage. That's the only +/// thing `T` is used for. The behavior of storing artifacts as files under +/// one or more paths is identical for all callers (i.e., both the real and +/// simulated sled agents). +/// +/// A given artifact is generally stored on both datasets designated for +/// artifact storage across both M.2 devices, but we attempt to be resilient to +/// a failing or missing M.2 device. This means: +/// +/// - for PUT, we try to write to all datasets, logging errors as we go; if we +/// successfully write the artifact to at least one, we return OK. +/// - for GET, we look in each dataset until we find it. +/// - for DELETE, we attempt to delete it from each dataset, logging errors as +/// we go, and failing if we saw any errors. +#[derive(Clone)] +pub(crate) struct ArtifactStore { + log: Logger, + reqwest_client: reqwest::Client, + storage: T, +} + +impl ArtifactStore { + pub(crate) fn new(log: &Logger, storage: T) -> ArtifactStore { + ArtifactStore { + log: log.new(slog::o!("component" => "ArtifactStore")), + reqwest_client: reqwest::ClientBuilder::new() + .connect_timeout(Duration::from_secs(15)) + .read_timeout(Duration::from_secs(15)) + .build() + .unwrap(), + storage, + } + } +} + +impl ArtifactStore { + pub(crate) async fn start( + self, + sled_address: SocketAddrV6, + dropshot_config: &ConfigDropshot, + ) -> Result>, StartError> + { + // In the real sled agent, the update datasets are durable and may + // retain temporary files leaked during a crash. Upon startup, we + // attempt to remove the subdirectory we store temporary files in, + // logging an error if that fails. + // + // (This function is part of `start` instead of `new` out of + // convenience: this function already needs to be async and fallible, + // but `new` doesn't; and all the sled agent implementations that don't + // call this function also don't need to run cleanup.) + for mountpoint in self + .storage + .artifact_storage_paths() + .await + .map_err(StartError::DatasetConfig)? + { + let path = mountpoint.join(TEMP_SUBDIR); + if let Err(err) = tokio::fs::remove_dir_all(&path).await { + if err.kind() != ErrorKind::NotFound { + // We log an error here because we expect that if we are + // having disk I/O errors, something else (fmd?) will + // identify those issues and bubble them up to the operator. + // (As of writing this comment that is not true but we + // expect this to exist in the limit, and refusing to start + // Sled Agent because of a problem with a single FRU seems + // inappropriate.) + error!( + &self.log, + "Failed to remove stale temporary artifacts"; + "error" => &err, + "path" => path.as_str(), + ); + } + } + } + + let mut depot_address = sled_address; + depot_address.set_port(REPO_DEPOT_PORT); + + let log = self.log.new(o!("component" => "dropshot (Repo Depot)")); + Ok(HttpServerStarter::new( + &ConfigDropshot { + bind_address: depot_address.into(), + ..dropshot_config.clone() + }, + repo_depot_api_mod::api_description::() + .expect("registered entrypoints"), + self, + &log, + ) + .map_err(StartError::Dropshot)? + .start()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum StartError { + #[error("Error retrieving dataset configuration")] + DatasetConfig(#[source] sled_storage::error::Error), + + #[error("Dropshot error while starting Repo Depot service")] + Dropshot(#[source] Box), +} + +macro_rules! log_and_store { + ($last_error:expr, $log:expr, $verb:literal, $path:expr, $err:expr) => {{ + error!( + $log, + concat!("Failed to ", $verb, " path"); + "error" => &$err, + "path" => $path.as_str(), + ); + $last_error = Some(Error::File { verb: $verb, path: $path, err: $err }); + }}; +} + +impl ArtifactStore { + /// GET operation (served by Repo Depot API) + /// + /// We try all datasets, returning early if we find the artifact, logging + /// errors as we go. If we don't find it we return the most recent error we + /// logged or a NotFound. + pub(crate) async fn get( + &self, + sha256: ArtifactHash, + ) -> Result { + let sha256_str = sha256.to_string(); + let mut last_error = None; + for mountpoint in self.storage.artifact_storage_paths().await? { + let path = mountpoint.join(&sha256_str); + match File::open(&path).await { + Ok(file) => { + info!( + &self.log, + "Retrieved artifact"; + "sha256" => &sha256_str, + "path" => path.as_str(), + ); + return Ok(file); + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => { + log_and_store!(last_error, &self.log, "open", path, err); + } + } + } + Err(last_error.unwrap_or(Error::NotFound { sha256 })) + } + + /// List operation (served by Sled Agent API) + /// + /// We try all datasets, logging errors as we go; if we're experiencing I/O + /// errors, Nexus should still be aware of the artifacts we think we have. + pub(crate) async fn list( + &self, + ) -> Result, Error> { + let mut map = BTreeMap::new(); + let mut any_datasets = false; + for mountpoint in self.storage.artifact_storage_paths().await? { + any_datasets = true; + let mut read_dir = match tokio::fs::read_dir(&mountpoint).await { + Ok(read_dir) => read_dir, + Err(err) => { + error!( + &self.log, + "Failed to read dir"; + "error" => &err, + "path" => mountpoint.as_str(), + ); + continue; + } + }; + // The semantics of tokio::fs::ReadDir are weird. At least with + // `std::fs::ReadDir`, we know when the end of the iterator is, + // because `.next()` returns `Option>`; we could + // theoretically log the error and continue trying to retrieve + // elements from the iterator (but whether this makes sense to do + // is not documented and likely system-dependent). + // + // The Tokio version returns `Result>`, which + // has no indication of whether there might be more items in + // the stream! (The stream adapter in tokio-stream simply calls + // `Result::transpose()`, so in theory an error is not the end of + // the stream.) + // + // For lack of any direction we stop reading entries from the stream + // on the first error. That way we at least don't get stuck retrying + // an operation that will always fail. + loop { + match read_dir.next_entry().await { + Ok(Some(entry)) => { + if let Ok(file_name) = entry.file_name().into_string() { + if let Ok(hash) = ArtifactHash::from_str(&file_name) + { + *map.entry(hash).or_default() += 1; + } + } + } + Ok(None) => break, + Err(err) => { + error!( + &self.log, + "Failed to read dir"; + "error" => &err, + "path" => mountpoint.as_str(), + ); + break; + } + } + } + } + if any_datasets { + Ok(map) + } else { + Err(Error::NoUpdateDataset) + } + } + + /// Common implementation for all artifact write operations that creates + /// a temporary file on all datasets. Returns an [`ArtifactWriter`] that + /// can be used to write the artifact to all temporary files, then move all + /// temporary files to their final paths. + /// + /// Most errors during the write process are considered non-fatal errors, + /// which are logged instead of immediately returned. + /// + /// In this method, possible fatal errors are: + /// - No temporary files could be created. + /// - A temporary file already exists (another task is writing to this + /// artifact). + async fn writer( + &self, + sha256: ArtifactHash, + ) -> Result { + let mut files = Vec::new(); + let mut last_error = None; + let mut datasets = 0; + for mountpoint in self.storage.artifact_storage_paths().await? { + datasets += 1; + let temp_dir = mountpoint.join(TEMP_SUBDIR); + if let Err(err) = tokio::fs::create_dir(&temp_dir).await { + if err.kind() != ErrorKind::AlreadyExists { + log_and_store!( + last_error, &self.log, "create", temp_dir, err + ); + continue; + } + } + + let temp_path = + Utf8TempPath::from_path(temp_dir.join(sha256.to_string())); + let file = match OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp_path) + .await + { + Ok(file) => file, + Err(err) => { + if err.kind() == ErrorKind::AlreadyExists { + return Err(Error::AlreadyInProgress { sha256 }); + } else { + let path = temp_path.to_path_buf(); + log_and_store!( + last_error, &self.log, "create", path, err + ); + continue; + } + } + }; + let file = NamedUtf8TempFile::from_parts(file, temp_path); + + files.push(Some((file, mountpoint))); + } + if files.is_empty() { + Err(last_error.unwrap_or(Error::NoUpdateDataset)) + } else { + Ok(ArtifactWriter { + datasets, + hasher: Sha256::new(), + files, + log: self.log.clone(), + sha256, + }) + } + } + + /// PUT operation (served by Sled Agent API) which takes a [`StreamingBody`] + pub(crate) async fn put_body( + &self, + sha256: ArtifactHash, + body: StreamingBody, + ) -> Result { + self.writer(sha256) + .await? + .write_stream(body.into_stream().map_err(Error::Body)) + .await + } + + /// POST operation (served by Sled Agent API) + pub(crate) async fn copy_from_depot( + &self, + sha256: ArtifactHash, + depot_base_url: &str, + ) -> Result<(), Error> { + let client = repo_depot_client::Client::new_with_client( + depot_base_url, + self.reqwest_client.clone(), + self.log.new(slog::o!( + "component" => "Repo Depot client (ArtifactStore)", + "base_url" => depot_base_url.to_owned(), + )), + ); + // Check that there's no conflict before we send the upstream request. + let writer = self.writer(sha256).await?; + let response = client + .artifact_get_by_sha256(&sha256.to_string()) + .await + .map_err(|err| Error::DepotCopy { + sha256, + base_url: depot_base_url.to_owned(), + err, + })?; + // Copy from the stream on its own task and immediately return. + let log = self.log.clone(); + let base_url = depot_base_url.to_owned(); + tokio::task::spawn(async move { + let stream = response.into_inner().into_inner().map_err(|err| { + Error::DepotCopy { + sha256, + base_url: base_url.clone(), + err: repo_depot_client::ClientError::ResponseBodyError(err), + } + }); + if let Err(err) = writer.write_stream(stream).await { + error!( + &log, + "Failed to write artifact"; + "err" => &err, + ); + } + }); + Ok(()) + } + + /// DELETE operation (served by Sled Agent API) + /// + /// We attempt to delete the artifact in all datasets, logging errors as we + /// go. If any errors occurred we return the most recent error we logged. + pub(crate) async fn delete( + &self, + sha256: ArtifactHash, + ) -> Result<(), Error> { + let sha256 = sha256.to_string(); + let mut any_datasets = false; + let mut last_error = None; + for mountpoint in self.storage.artifact_storage_paths().await? { + any_datasets = true; + let path = mountpoint.join(&sha256); + match tokio::fs::remove_file(&path).await { + Ok(()) => { + info!( + &self.log, + "Removed artifact"; + "sha256" => &sha256, + "path" => path.as_str(), + ); + } + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => { + log_and_store!(last_error, &self.log, "remove", path, err); + } + } + } + if let Some(last_error) = last_error { + Err(last_error) + } else if any_datasets { + Ok(()) + } else { + // If we're here because there aren't any update datasets, we should + // report Service Unavailable instead of a successful result. + Err(Error::NoUpdateDataset) + } + } +} + +/// Abstracts over what kind of sled agent we are; each of the real sled agent, +/// simulated sled agent, and this module's unit tests have different ways of +/// keeping track of the datasets on the system. +pub(crate) trait DatasetsManager: Sync { + async fn artifact_storage_paths( + &self, + ) -> Result + '_, StorageError>; +} + +/// Iterator `.filter().map()` common to `DatasetsManager` implementations. +pub(crate) fn filter_dataset_mountpoints( + config: DatasetsConfig, + root: &Utf8Path, +) -> impl Iterator + '_ { + config + .datasets + .into_values() + .filter(|dataset| *dataset.name.dataset() == DatasetKind::Update) + .map(|dataset| dataset.name.mountpoint(root)) +} + +impl DatasetsManager for StorageHandle { + async fn artifact_storage_paths( + &self, + ) -> Result + '_, StorageError> { + // TODO: When datasets are managed by Reconfigurator (#6229), + // this should be changed to use `self.datasets_config_list()` and + // `filter_dataset_mountpoints`. + Ok(self + .get_latest_disks() + .await + .all_m2_mountpoints(M2_ARTIFACT_DATASET) + .into_iter()) + } +} + +/// Abstraction that handles writing to several temporary files. +struct ArtifactWriter { + datasets: usize, + files: Vec, Utf8PathBuf)>>, + hasher: Sha256, + log: Logger, + sha256: ArtifactHash, +} + +impl ArtifactWriter { + /// Calls [`ArtifactWriter::write`] for each chunk in the stream, then + /// [`ArtifactWriter::finalize`]. See the documentation for these functions + /// for error handling information. + async fn write_stream( + self, + stream: impl Stream, Error>>, + ) -> Result { + let writer = stream + .try_fold(self, |mut writer, chunk| async { + writer.write(chunk).await?; + Ok(writer) + }) + .await?; + writer.finalize().await + } + + /// Write `chunk` to all temporary files. + /// + /// Errors in this method are considered non-fatal errors. All errors + /// are logged. If all files have failed, this method returns the most + /// recently-seen non-fatal error as a fatal error. + async fn write(&mut self, chunk: impl AsRef<[u8]>) -> Result<(), Error> { + self.hasher.update(&chunk); + + let mut last_error = None; + for option in &mut self.files { + if let Some((mut file, mountpoint)) = option.take() { + match file.as_file_mut().write_all(chunk.as_ref()).await { + Ok(()) => { + *option = Some((file, mountpoint)); + } + Err(err) => { + let path = file.path().to_owned(); + log_and_store!( + last_error, &self.log, "write to", path, err + ); + // `file` and `final_path` are dropped here, cleaning up + // the file + } + } + } + } + + self.files.retain(Option::is_some); + if self.files.is_empty() { + Err(last_error.unwrap_or(Error::NoUpdateDataset)) + } else { + Ok(()) + } + } + + /// Rename all files to their final paths. + /// + /// Errors in this method are considered non-fatal errors. If all files have + /// failed in some way, the most recently-seen error is returned as a fatal + /// error. + async fn finalize(self) -> Result { + let digest = self.hasher.finalize(); + if digest.as_slice() != self.sha256.as_ref() { + return Err(Error::HashMismatch { + expected: self.sha256, + actual: ArtifactHash(digest.into()), + }); + } + + let mut last_error = None; + let mut successful_writes = 0; + for (mut file, mountpoint) in self.files.into_iter().flatten() { + // 1. fsync the temporary file. + if let Err(err) = file.as_file_mut().sync_all().await { + let path = file.path().to_owned(); + log_and_store!(last_error, &self.log, "sync", path, err); + continue; + } + // 2. Open the parent directory so we can fsync it. + let parent_dir = match File::open(&mountpoint).await { + Ok(dir) => dir, + Err(err) => { + log_and_store!( + last_error, &self.log, "open", mountpoint, err + ); + continue; + } + }; + // 3. Rename the temporary file. + let final_path = mountpoint.join(self.sha256.to_string()); + let moved_final_path = final_path.clone(); + if let Err(err) = tokio::task::spawn_blocking(move || { + file.persist(&moved_final_path) + }) + .await? + { + error!( + &self.log, + "Failed to rename temporary file"; + "error" => &err.error, + "from" => err.file.path().as_str(), + "to" => final_path.as_str(), + ); + last_error = Some(Error::FileRename { + from: err.file.path().to_owned(), + to: final_path, + err: err.error, + }); + continue; + } + // 4. fsync the parent directory. + if let Err(err) = parent_dir.sync_all().await { + log_and_store!(last_error, &self.log, "sync", mountpoint, err); + continue; + } + + successful_writes += 1; + } + + if successful_writes > 0 { + info!( + &self.log, + "Wrote artifact"; + "sha256" => &self.sha256.to_string(), + "datasets" => self.datasets, + "successful_writes" => successful_writes, + ); + Ok(ArtifactPutResponse { + datasets: self.datasets, + successful_writes, + }) + } else { + Err(last_error.unwrap_or(Error::NoUpdateDataset)) + } + } +} + +/// Implementation of the Repo Depot API backed by an +/// `ArtifactStore`. +enum RepoDepotImpl {} + +impl RepoDepotApi for RepoDepotImpl { + type Context = ArtifactStore; + + async fn artifact_get_by_sha256( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let sha256 = path_params.into_inner().sha256; + let file = rqctx.context().get(sha256).await?; + let file_access = hyper_staticfile::vfs::TokioFileAccess::new(file); + let file_stream = + hyper_staticfile::util::FileBytesStream::new(file_access); + let body = Body::wrap(hyper_staticfile::Body::Full(file_stream)); + Ok(HttpResponseOk(FreeformBody(body))) + } +} + +#[derive(Debug, thiserror::Error, SlogInlineError)] +pub(crate) enum Error { + #[error("Another task is already writing artifact {sha256}")] + AlreadyInProgress { sha256: ArtifactHash }, + + #[error("Error while reading request body")] + Body(dropshot::HttpError), + + #[error("Error retrieving dataset configuration")] + DatasetConfig(#[from] sled_storage::error::Error), + + #[error("Error fetching artifact {sha256} from depot at {base_url}")] + DepotCopy { + sha256: ArtifactHash, + base_url: String, + #[source] + err: repo_depot_client::ClientError, + }, + + #[error("Failed to {verb} `{path}`")] + File { + verb: &'static str, + path: Utf8PathBuf, + #[source] + err: std::io::Error, + }, + + #[error("Failed to rename `{from}` to `{to}`")] + FileRename { + from: Utf8PathBuf, + to: Utf8PathBuf, + #[source] + err: std::io::Error, + }, + + #[error("Digest mismatch: expected {expected}, actual {actual}")] + HashMismatch { expected: ArtifactHash, actual: ArtifactHash }, + + #[error("Blocking task failed")] + Join(#[from] tokio::task::JoinError), + + #[error("Artifact {sha256} not found")] + NotFound { sha256: ArtifactHash }, + + #[error("No update datasets present")] + NoUpdateDataset, +} + +impl From for HttpError { + fn from(err: Error) -> HttpError { + match err { + Error::AlreadyInProgress { .. } => HttpError::for_client_error( + None, + StatusCode::CONFLICT, + err.to_string(), + ), + Error::Body(inner) => inner, + Error::DatasetConfig(_) | Error::NoUpdateDataset => { + HttpError::for_unavail(None, err.to_string()) + } + Error::DepotCopy { .. } + | Error::File { .. } + | Error::FileRename { .. } + | Error::Join(_) => HttpError::for_internal_error(err.to_string()), + Error::HashMismatch { .. } => { + HttpError::for_bad_request(None, err.to_string()) + } + Error::NotFound { .. } => { + HttpError::for_not_found(None, err.to_string()) + } + } + } +} + +#[cfg(test)] +mod test { + use camino_tempfile::Utf8TempDir; + use futures::stream; + use hex_literal::hex; + use omicron_common::disk::{ + DatasetConfig, DatasetKind, DatasetName, DatasetsConfig, + }; + use omicron_common::update::ArtifactHash; + use omicron_common::zpool_name::ZpoolName; + use omicron_test_utils::dev::test_setup_log; + use omicron_uuid_kinds::{DatasetUuid, ZpoolUuid}; + use sled_storage::error::Error as StorageError; + use tokio::io::AsyncReadExt; + + use super::{ArtifactStore, DatasetsManager, Error}; + + struct TestBackend { + datasets: DatasetsConfig, + mountpoint_root: Utf8TempDir, + } + + impl TestBackend { + fn new(len: usize) -> TestBackend { + let mountpoint_root = camino_tempfile::tempdir().unwrap(); + + let mut datasets = DatasetsConfig::default(); + if len > 0 { + datasets.generation = datasets.generation.next(); + } + for _ in 0..len { + let dataset = DatasetConfig { + id: DatasetUuid::new_v4(), + name: DatasetName::new( + ZpoolName::new_external(ZpoolUuid::new_v4()), + DatasetKind::Update, + ), + compression: Default::default(), + quota: None, + reservation: None, + }; + let mountpoint = + dataset.name.mountpoint(mountpoint_root.path()); + std::fs::create_dir_all(mountpoint).unwrap(); + datasets.datasets.insert(dataset.id, dataset); + } + + TestBackend { datasets, mountpoint_root } + } + } + + impl DatasetsManager for TestBackend { + async fn artifact_storage_paths( + &self, + ) -> Result + '_, StorageError> + { + Ok(super::filter_dataset_mountpoints( + self.datasets.clone(), + self.mountpoint_root.path(), + )) + } + } + + const TEST_ARTIFACT: &[u8] = b"I'm an artifact!\n"; + const TEST_HASH: ArtifactHash = ArtifactHash(hex!( + "ab3581cd62f6645518f61a8e4391af6c062d5d60111edb0e51b37bd84827f5b4" + )); + + #[tokio::test] + async fn list_get_put_delete() { + let log = test_setup_log("get_put_delete"); + let backend = TestBackend::new(2); + let store = ArtifactStore::new(&log.log, backend); + + // list succeeds with an empty result + assert!(store.list().await.unwrap().is_empty()); + // get fails, because it doesn't exist yet + assert!(matches!( + store.get(TEST_HASH).await, + Err(Error::NotFound { .. }) + )); + // delete does not fail because we don't fail if the artifact is not + // present + assert!(matches!(store.delete(TEST_HASH).await, Ok(()))); + + // test several things here: + // 1. put succeeds + // 2. put is idempotent (we don't care if it clobbers a file as long as + // the hash is okay) + // 3. we don't fail trying to create TEMP_SUBDIR twice + for _ in 0..2 { + store + .writer(TEST_HASH) + .await + .unwrap() + .write_stream(stream::once(async { Ok(TEST_ARTIFACT) })) + .await + .unwrap(); + // list lists the file + assert!(store + .list() + .await + .unwrap() + .into_iter() + .eq([(TEST_HASH, 2)])); + // get succeeds, file reads back OK + let mut file = store.get(TEST_HASH).await.unwrap(); + let mut vec = Vec::new(); + file.read_to_end(&mut vec).await.unwrap(); + assert_eq!(vec, TEST_ARTIFACT); + } + + // all datasets should have the artifact + for mountpoint in store.storage.artifact_storage_paths().await.unwrap() + { + assert_eq!( + tokio::fs::read(mountpoint.join(TEST_HASH.to_string())) + .await + .unwrap(), + TEST_ARTIFACT + ); + } + + // delete succeeds and is idempotent + for _ in 0..2 { + store.delete(TEST_HASH).await.unwrap(); + // list succeeds with an empty result + assert!(store.list().await.unwrap().is_empty()); + // get now fails because it no longer exists + assert!(matches!( + store.get(TEST_HASH).await, + Err(Error::NotFound { .. }) + )); + } + + log.cleanup_successful(); + } + + #[tokio::test] + async fn no_dataset() { + // If there are no update datasets, all gets should fail with + // `Error::NotFound`, and all other operations should fail with + // `Error::NoUpdateDataset`. + + let log = test_setup_log("no_dataset"); + let backend = TestBackend::new(0); + let store = ArtifactStore::new(&log.log, backend); + + assert!(matches!( + store.writer(TEST_HASH).await, + Err(Error::NoUpdateDataset) + )); + assert!(matches!( + store.get(TEST_HASH).await, + Err(Error::NotFound { .. }) + )); + assert!(matches!(store.list().await, Err(Error::NoUpdateDataset))); + assert!(matches!( + store.delete(TEST_HASH).await, + Err(Error::NoUpdateDataset) + )); + + log.cleanup_successful(); + } + + #[tokio::test] + async fn wrong_hash() { + const ACTUAL: ArtifactHash = ArtifactHash(hex!( + "4d27a9d1ddb65e0f2350a400cf73157e42ae2ca687a4220aa0a73b9bb2d211f7" + )); + + let log = test_setup_log("wrong_hash"); + let backend = TestBackend::new(2); + let store = ArtifactStore::new(&log.log, backend); + let err = store + .writer(TEST_HASH) + .await + .unwrap() + .write_stream(stream::once(async { + Ok(b"This isn't right at all.") + })) + .await + .unwrap_err(); + match err { + Error::HashMismatch { expected, actual } => { + assert_eq!(expected, TEST_HASH); + assert_eq!(actual, ACTUAL); + } + err => panic!("wrong error: {err}"), + } + assert!(matches!( + store.get(TEST_HASH).await, + Err(Error::NotFound { .. }) + )); + + log.cleanup_successful(); + } +} diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 9685780a0e..acfbafe61c 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -6,7 +6,7 @@ use anyhow::{anyhow, Context}; use dpd_client::types::{ - LinkCreate, LinkId, LinkSettings, PortId, PortSettings, + LinkCreate, LinkId, LinkSettings, PortId, PortSettings, TxEq, }; use dpd_client::Client as DpdClient; use futures::future; @@ -702,6 +702,13 @@ impl<'a> EarlyNetworkSetup<'a> { fec: convert_fec(&port_config.uplink_port_fec), speed: convert_speed(&port_config.uplink_port_speed), lane: Some(LinkId(0)), + tx_eq: port_config.tx_eq.map(|x| TxEq { + pre1: x.pre1, + pre2: x.pre2, + main: x.main, + post2: x.post2, + post1: x.post1, + }), }, addrs, }; diff --git a/sled-agent/src/common/disk.rs b/sled-agent/src/common/disk.rs index 7bef28ac7c..ba50d0f7c1 100644 --- a/sled-agent/src/common/disk.rs +++ b/sled-agent/src/common/disk.rs @@ -8,7 +8,6 @@ use chrono::Utc; use omicron_common::api::external::DiskState; use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::DiskRuntimeState; -use propolis_client::types::DiskAttachmentState as PropolisDiskState; use sled_agent_types::disk::DiskStateRequested; use uuid::Uuid; @@ -47,15 +46,9 @@ impl DiskStates { /// Propolis. pub fn observe_transition( &mut self, - observed: &PropolisDiskState, + observed: &DiskState, ) -> Option { - let next = match observed { - PropolisDiskState::Attached(uuid) => DiskState::Attached(*uuid), - PropolisDiskState::Detached => DiskState::Detached, - PropolisDiskState::Destroyed => DiskState::Destroyed, - PropolisDiskState::Faulted => DiskState::Faulted, - }; - self.transition(next, None); + self.transition(observed.clone(), None); None } diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 489fc9ab0d..cd3afc44ac 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -11,10 +11,10 @@ use bootstore::schemes::v0::NetworkConfig; use camino::Utf8PathBuf; use display_error_chain::DisplayErrorChain; use dropshot::{ - ApiDescription, Body, FreeformBody, HttpError, HttpResponseCreated, - HttpResponseDeleted, HttpResponseHeaders, HttpResponseOk, - HttpResponseUpdatedNoContent, Path, Query, RequestContext, StreamingBody, - TypedBody, + ApiDescription, Body, FreeformBody, HttpError, HttpResponseAccepted, + HttpResponseCreated, HttpResponseDeleted, HttpResponseHeaders, + HttpResponseOk, HttpResponseUpdatedNoContent, Path, Query, RequestContext, + StreamingBody, TypedBody, }; use nexus_sled_agent_shared::inventory::{ Inventory, OmicronZonesConfig, SledRole, @@ -31,6 +31,7 @@ use omicron_common::disk::{ DatasetsConfig, DatasetsManagementResult, DiskVariant, DisksManagementResult, M2Slot, OmicronPhysicalDisksConfig, }; +use omicron_common::update::ArtifactHash; use sled_agent_api::*; use sled_agent_types::boot_disk::{ BootDiskOsWriteStatus, BootDiskPathParams, BootDiskUpdatePathParams, @@ -401,6 +402,48 @@ impl SledAgentApi for SledAgentImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn artifact_list( + rqctx: RequestContext, + ) -> Result>, HttpError> { + Ok(HttpResponseOk(rqctx.context().artifact_store().list().await?)) + } + + async fn artifact_copy_from_depot( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError> + { + let sha256 = path_params.into_inner().sha256; + let depot_base_url = body.into_inner().depot_base_url; + rqctx + .context() + .artifact_store() + .copy_from_depot(sha256, &depot_base_url) + .await?; + Ok(HttpResponseAccepted(ArtifactCopyFromDepotResponse {})) + } + + async fn artifact_put( + rqctx: RequestContext, + path_params: Path, + body: StreamingBody, + ) -> Result, HttpError> { + let sha256 = path_params.into_inner().sha256; + Ok(HttpResponseOk( + rqctx.context().artifact_store().put_body(sha256, body).await?, + )) + } + + async fn artifact_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let sha256 = path_params.into_inner().sha256; + rqctx.context().artifact_store().delete(sha256).await?; + Ok(HttpResponseDeleted()) + } + async fn vmm_issue_disk_snapshot_request( rqctx: RequestContext, path_params: Path, diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index b47aeb6508..4883918c36 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -32,7 +32,9 @@ use omicron_common::backoff; use omicron_common::backoff::BackoffError; use omicron_common::zpool_name::ZpoolName; use omicron_common::NoDebug; -use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid}; +use omicron_uuid_kinds::{ + GenericUuid, InstanceUuid, OmicronZoneUuid, PropolisUuid, +}; use propolis_api_types::ErrorCode as PropolisErrorCode; use propolis_client::Client as PropolisClient; use rand::prelude::IteratorRandom; @@ -232,10 +234,6 @@ enum InstanceRequest { state: VmmStateRequested, tx: oneshot::Sender>, }, - Terminate { - mark_failed: bool, - tx: oneshot::Sender>, - }, IssueSnapshotRequest { disk_id: Uuid, snapshot_id: Uuid, @@ -293,9 +291,6 @@ impl InstanceRequest { Self::PutState { tx, .. } => tx .send(Err(error.into())) .map_err(|_| Error::FailedSendClientClosed), - Self::Terminate { tx, .. } => tx - .send(Err(error.into())) - .map_err(|_| Error::FailedSendClientClosed), Self::IssueSnapshotRequest { tx, .. } | Self::AddExternalIp { tx, .. } | Self::DeleteExternalIp { tx, .. } @@ -306,6 +301,11 @@ impl InstanceRequest { } } +struct TerminateRequest { + mark_failed: bool, + tx: oneshot::Sender>, +} + // A small task which tracks the state of the instance, by constantly querying // the state of Propolis for updates. // @@ -418,6 +418,8 @@ struct InstanceRunner { // Properties visible to Propolis properties: propolis_client::types::InstanceProperties, + vcpus: u8, + memory_mib: u64, // The ID of the Propolis server (and zone) running this instance propolis_id: PropolisUuid, @@ -469,7 +471,7 @@ struct InstanceRunner { } impl InstanceRunner { - async fn run(mut self) { + async fn run(mut self, mut terminate_rx: mpsc::Receiver) { use InstanceRequest::*; while !self.should_terminate { tokio::select! { @@ -535,74 +537,103 @@ impl InstanceRunner { self.terminate(mark_failed).await; }, } - }, + // Requests to terminate the instance take priority over any + // other request to the instance. + request = terminate_rx.recv() => { + self.handle_termination_request(request, None).await; + break; + } + // Handle external requests to act upon the instance. request = self.rx.recv() => { - let request_variant = request.as_ref().map(|r| r.to_string()); - let result = match request { - Some(RequestZoneBundle { tx }) => { - tx.send(self.request_zone_bundle().await) - .map_err(|_| Error::FailedSendClientClosed) - }, - Some(GetFilesystemPool { tx } ) => { - tx.send(Ok(self.get_filesystem_zpool())) - .map_err(|_| Error::FailedSendClientClosed) - }, - Some(CurrentState{ tx }) => { - tx.send(Ok(self.current_state())) - .map_err(|_| Error::FailedSendClientClosed) - }, - Some(PutState{ state, tx }) => { - tx.send(self.put_state(state).await - .map(|r| VmmPutStateResponse { updated_runtime: Some(r) }) - .map_err(|e| e.into())) - .map_err(|_| Error::FailedSendClientClosed) - }, - Some(Terminate { mark_failed, tx }) => { - tx.send(Ok(VmmUnregisterResponse { - updated_runtime: Some(self.terminate(mark_failed).await) - })) - .map_err(|_| Error::FailedSendClientClosed) - }, - Some(IssueSnapshotRequest { disk_id, snapshot_id, tx }) => { - tx.send( - self.issue_snapshot_request( - disk_id, - snapshot_id - ).await.map_err(|e| e.into()) - ) - .map_err(|_| Error::FailedSendClientClosed) - }, - Some(AddExternalIp { ip, tx }) => { - tx.send(self.add_external_ip(&ip).await.map_err(|e| e.into())) - .map_err(|_| Error::FailedSendClientClosed) - }, - Some(DeleteExternalIp { ip, tx }) => { - tx.send(self.delete_external_ip(&ip).await.map_err(|e| e.into())) - .map_err(|_| Error::FailedSendClientClosed) - }, - Some(RefreshExternalIps { tx }) => { - tx.send(self.refresh_external_ips().map_err(|e| e.into())) - .map_err(|_| Error::FailedSendClientClosed) - } + let request = match request { + Some(r) => r, None => { warn!(self.log, "Instance request channel closed; shutting down"); let mark_failed = false; self.terminate(mark_failed).await; break; - }, + } }; + let request_variant = request.to_string(); + // Okay, this is a little bit wacky: if we are waiting for + // one of the instance operations we run here to come back, + // and a termination request comes in, we want to give up on + // the outstanding operation and honor the termination + // request immediately. This is in case the instance + // operation has gotten stuck: we don't want it to prevent + // the instance from terminating because something else is + // wedged. + // + // Therefore, we're going to select between the future that + // actually performs the instance op and receiving another + // request from the termination channel. + let op = async { + match request { + RequestZoneBundle { tx } => { + tx.send(self.request_zone_bundle().await) + .map_err(|_| Error::FailedSendClientClosed) + }, + GetFilesystemPool { tx } => { + tx.send(Ok(self.get_filesystem_zpool())) + .map_err(|_| Error::FailedSendClientClosed) + }, + CurrentState{ tx } => { + tx.send(Ok(self.current_state())) + .map_err(|_| Error::FailedSendClientClosed) + }, + PutState{ state, tx } => { + tx.send(self.put_state(state).await + .map(|r| VmmPutStateResponse { updated_runtime: Some(r) }) + .map_err(|e| e.into())) + .map_err(|_| Error::FailedSendClientClosed) + }, + IssueSnapshotRequest { disk_id, snapshot_id, tx } => { + tx.send( + self.issue_snapshot_request( + disk_id, + snapshot_id + ).await.map_err(|e| e.into()) + ) + .map_err(|_| Error::FailedSendClientClosed) + }, + AddExternalIp { ip, tx } => { + tx.send(self.add_external_ip(&ip).await.map_err(|e| e.into())) + .map_err(|_| Error::FailedSendClientClosed) + }, + DeleteExternalIp { ip, tx } => { + tx.send(self.delete_external_ip(&ip).await.map_err(|e| e.into())) + .map_err(|_| Error::FailedSendClientClosed) + }, + RefreshExternalIps { tx } => { + tx.send(self.refresh_external_ips().map_err(|e| e.into())) + .map_err(|_| Error::FailedSendClientClosed) + } + } + }; + tokio::select! { + biased; + + request = terminate_rx.recv() => { + self.handle_termination_request( + request, + Some(&request_variant), + ).await; + break; + } - if let Err(err) = result { - warn!( - self.log, - "Error handling request"; - "request" => request_variant.unwrap(), - "err" => ?err, - - ); - } + result = op => { + if let Err(err) = result { + warn!( + self.log, + "Error handling request"; + "request" => request_variant, + "err" => ?err, + ); + } + } + }; } } @@ -627,9 +658,6 @@ impl InstanceRunner { PutState { tx, .. } => { tx.send(Err(Error::Terminating.into())).map_err(|_| ()) } - Terminate { tx, .. } => { - tx.send(Err(Error::Terminating.into())).map_err(|_| ()) - } IssueSnapshotRequest { tx, .. } => { tx.send(Err(Error::Terminating.into())).map_err(|_| ()) } @@ -644,6 +672,15 @@ impl InstanceRunner { } }; } + + // Anyone else who was trying to ask us to go die will be happy to learn + // that we have now done so! + while let Some(TerminateRequest { tx, .. }) = terminate_rx.recv().await + { + let _ = tx.send(Ok(VmmUnregisterResponse { + updated_runtime: Some(self.current_state()), + })); + } } /// Yields this instance's ID. @@ -876,6 +913,8 @@ impl InstanceRunner { let request = propolis_client::types::InstanceEnsureRequest { properties: self.properties.clone(), + vcpus: self.vcpus, + memory: self.memory_mib, nics, disks: self .requested_disks @@ -1046,7 +1085,7 @@ impl InstanceRunner { } } - let Some(primary_nic) = self.requested_nics.get(0) else { + let Some(primary_nic) = self.primary_nic() else { return Err(Error::Opte(illumos_utils::opte::Error::NoPrimaryNic)); }; @@ -1062,7 +1101,7 @@ impl InstanceRunner { } fn refresh_external_ips_inner(&mut self) -> Result<(), Error> { - let Some(primary_nic) = self.requested_nics.get(0) else { + let Some(primary_nic) = self.primary_nic() else { return Err(Error::Opte(illumos_utils::opte::Error::NoPrimaryNic)); }; @@ -1110,7 +1149,7 @@ impl InstanceRunner { } } - let Some(primary_nic) = self.requested_nics.get(0) else { + let Some(primary_nic) = self.primary_nic() else { return Err(Error::Opte(illumos_utils::opte::Error::NoPrimaryNic)); }; @@ -1124,6 +1163,10 @@ impl InstanceRunner { Ok(()) } + + fn primary_nic(&self) -> Option<&NetworkInterface> { + self.requested_nics.iter().find(|nic| nic.primary) + } } fn propolis_error_code( @@ -1189,6 +1232,12 @@ pub struct Instance { /// loop. tx: mpsc::Sender, + /// Sender for requests to terminate the instance. + /// + /// These are sent over a separate channel so that they can be prioritized + /// over all other requests to the instance. + terminate_tx: mpsc::Sender, + /// This is reference-counted so that the `Instance` struct may be cloned. #[allow(dead_code)] runner_handle: Arc>, @@ -1287,6 +1336,19 @@ impl Instance { let (tx, rx) = mpsc::channel(QUEUE_SIZE); let (tx_monitor, rx_monitor) = mpsc::channel(1); + // Request channel for terminating the instance. + // + // This is a separate channel from the main request channel (`self.rx`) + // because we would like to be able to prioritize requests to terminate, and + // handle them even when the instance's main request channel may have filled + // up. + // + // Note also that this is *not* part of the `InstanceRunner` struct, + // because it's necessary to split mutable borrows in order to allow + // selecting between the actual instance operation (which must mutate + // the `InstanceRunner`) and awaiting a termination request. + let (terminate_tx, terminate_rx) = mpsc::channel(QUEUE_SIZE); + let metadata = propolis_client::types::InstanceMetadata { project_id: metadata.project_id, silo_id: metadata.silo_id, @@ -1308,15 +1370,13 @@ impl Instance { id: id.into_untyped_uuid(), name: hardware.properties.hostname.to_string(), description: "Test description".to_string(), - image_id: Uuid::nil(), - bootrom_id: Uuid::nil(), - // TODO: Align the byte type w/propolis. - memory: hardware.properties.memory.to_whole_mebibytes(), - // TODO: we should probably make propolis aligned with - // InstanceCpuCount here, to avoid any casting... - vcpus: hardware.properties.ncpus.0 as u8, metadata, }, + // TODO: we should probably make propolis aligned with + // InstanceCpuCount here, to avoid any casting... + vcpus: hardware.properties.ncpus.0 as u8, + // TODO: Align the byte type w/propolis. + memory_mib: hardware.properties.memory.to_whole_mebibytes(), propolis_id, propolis_addr, vnic_allocator, @@ -1341,9 +1401,14 @@ impl Instance { }; let runner_handle = - tokio::task::spawn(async move { runner.run().await }); + tokio::task::spawn(async move { runner.run(terminate_rx).await }); - Ok(Instance { id, tx, runner_handle: Arc::new(runner_handle) }) + Ok(Instance { + id, + tx, + runner_handle: Arc::new(runner_handle), + terminate_tx, + }) } pub fn id(&self) -> InstanceUuid { @@ -1406,9 +1471,19 @@ impl Instance { tx: oneshot::Sender>, mark_failed: bool, ) -> Result<(), Error> { - self.tx - .try_send(InstanceRequest::Terminate { mark_failed, tx }) - .or_else(InstanceRequest::fail_try_send) + self.terminate_tx + .try_send(TerminateRequest { mark_failed, tx }) + .or_else(|err| match err { + mpsc::error::TrySendError::Closed(TerminateRequest { + tx, + .. + }) => tx.send(Err(Error::FailedSendChannelClosed.into())), + mpsc::error::TrySendError::Full(TerminateRequest { + tx, + .. + }) => tx.send(Err(Error::FailedSendChannelFull.into())), + }) + .map_err(|_| Error::FailedSendClientClosed) } pub fn issue_snapshot_request( @@ -1661,7 +1736,9 @@ impl InstanceRunner { .with_zone_root_path(root) .with_zone_image_paths(&["/opt/oxide".into()]) .with_zone_type("propolis-server") - .with_unique_name(self.propolis_id.into_untyped_uuid()) + .with_unique_name(OmicronZoneUuid::from_untyped_uuid( + self.propolis_id.into_untyped_uuid(), + )) .with_datasets(&[]) .with_filesystems(&[]) .with_data_links(&[]) @@ -1745,6 +1822,64 @@ impl InstanceRunner { Ok(PropolisSetup { client, running_zone }) } + async fn handle_termination_request( + &mut self, + req: Option, + current_req: Option<&str>, + ) { + match req { + Some(TerminateRequest { tx, mark_failed }) => { + if let Some(request) = current_req { + info!( + self.log, + "Received request to terminate instance while waiting \ + on an ongoing request"; + "request" => %request, + "mark_failed" => mark_failed, + ); + } else { + info!( + self.log, + "Received request to terminate instance"; + "mark_failed" => mark_failed, + ); + } + + let result = tx + .send(Ok(VmmUnregisterResponse { + updated_runtime: Some( + self.terminate(mark_failed).await, + ), + })) + .map_err(|_| Error::FailedSendClientClosed); + if let Err(err) = result { + warn!( + self.log, + "Error handling request to terminate instance"; + "err" => ?err, + ); + } + } + None => { + if let Some(request) = current_req { + warn!( + self.log, + "Instance termination request channel closed while \ + waiting on an ongoing request; shutting down"; + "request" => %request, + ); + } else { + warn!( + self.log, + "Instance termination request channel closed; \ + shutting down"; + ); + } + self.terminate(false).await; + } + }; + } + async fn terminate(&mut self, mark_failed: bool) -> SledVmmState { self.terminate_inner().await; self.state.terminate_rudely(mark_failed); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 34fb8e493d..82099763ea 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -146,7 +146,7 @@ impl InstanceManager { .tx .send(InstanceManagerRequest::EnsureRegistered { propolis_id, - instance, + instance: Box::new(instance), sled_identifiers: Box::new(sled_identifiers), tx, }) @@ -335,12 +335,12 @@ impl InstanceManager { enum InstanceManagerRequest { EnsureRegistered { propolis_id: PropolisUuid, - instance: InstanceEnsureBody, // These are boxed because they are, apparently, quite large, and Clippy // whinges about the overall size of this variant relative to the // others. Since we will generally send `EnsureRegistered` requests much // less frequently than most of the others, boxing this seems like a // reasonable choice... + instance: Box, sled_identifiers: Box, tx: oneshot::Sender>, }, @@ -462,7 +462,7 @@ impl InstanceManagerRunner { sled_identifiers, tx, }) => { - tx.send(self.ensure_registered(propolis_id, instance, *sled_identifiers).await).map_err(|_| Error::FailedSendClientClosed) + tx.send(self.ensure_registered(propolis_id, *instance, *sled_identifiers).await).map_err(|_| Error::FailedSendClientClosed) }, Some(EnsureUnregistered { propolis_id, tx }) => { self.ensure_unregistered(tx, propolis_id) diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index a2421528e2..b2d78c4a5e 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -15,6 +15,7 @@ pub mod sim; pub mod common; // Modules for the non-simulated sled agent. +pub mod artifact_store; mod backing_fs; mod boot_disk_os_writer; pub mod bootstrap; diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index f9c0f117ba..de0b086752 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -4,7 +4,6 @@ use nexus_sled_agent_shared::inventory::{OmicronZoneConfig, OmicronZoneType}; use omicron_common::disk::{DatasetKind, DatasetName}; -use omicron_uuid_kinds::GenericUuid; pub use sled_hardware::DendriteAsic; use std::net::SocketAddrV6; @@ -20,7 +19,7 @@ impl OmicronZoneConfigExt for OmicronZoneConfig { fn zone_name(&self) -> String { illumos_utils::running_zone::InstalledZone::get_zone_name( self.zone_type.kind().zone_prefix(), - Some(self.id.into_untyped_uuid()), + Some(self.id), ) } } diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index 42186f66e9..fb1399a9c2 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -16,6 +16,7 @@ use omicron_common::api::external::{ use omicron_common::api::internal::shared::{ NetworkInterface, ResolvedVpcFirewallRule, }; +use omicron_uuid_kinds::{GenericUuid, OmicronZoneUuid}; use rand::prelude::IteratorRandom; use rand::SeedableRng; use sled_storage::dataset::ZONE_DATASET; @@ -330,7 +331,7 @@ impl ProbeManagerInner { .with_zone_root_path(zone_root_path) .with_zone_image_paths(&["/opt/oxide".into()]) .with_zone_type("probe") - .with_unique_name(probe.id) + .with_unique_name(OmicronZoneUuid::from_untyped_uuid(probe.id)) .with_datasets(&[]) .with_filesystems(&[]) .with_data_links(&[]) diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index a963375381..2777545dc4 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -34,8 +34,8 @@ use omicron_common::backoff::{ retry_notify_ext, retry_policy_internal_service_aggressive, BackoffError, }; use omicron_common::disk::{ - DatasetKind, DatasetName, DiskVariant, OmicronPhysicalDiskConfig, - OmicronPhysicalDisksConfig, + DatasetKind, DatasetName, DatasetsConfig, DiskVariant, + OmicronPhysicalDiskConfig, OmicronPhysicalDisksConfig, }; use omicron_common::ledger::{self, Ledger, Ledgerable}; use omicron_common::policy::{ @@ -99,6 +99,12 @@ pub enum PlanError { #[error("Found only v2 service plan")] FoundV2, + + #[error("Found only v3 service plan")] + FoundV3, + + #[error("Found only v4 service plan")] + FoundV4, } #[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] @@ -106,6 +112,9 @@ pub struct SledConfig { /// Control plane disks configured for this sled pub disks: BlueprintPhysicalDisksConfig, + /// Datasets configured for this sled + pub datasets: DatasetsConfig, + /// zones configured for this sled pub zones: Vec, } @@ -125,7 +134,8 @@ impl Ledgerable for Plan { const RSS_SERVICE_PLAN_V1_FILENAME: &str = "rss-service-plan.json"; const RSS_SERVICE_PLAN_V2_FILENAME: &str = "rss-service-plan-v2.json"; const RSS_SERVICE_PLAN_V3_FILENAME: &str = "rss-service-plan-v3.json"; -const RSS_SERVICE_PLAN_FILENAME: &str = "rss-service-plan-v4.json"; +const RSS_SERVICE_PLAN_V4_FILENAME: &str = "rss-service-plan-v4.json"; +const RSS_SERVICE_PLAN_FILENAME: &str = "rss-service-plan-v5.json"; pub fn from_sockaddr_to_external_floating_addr( addr: SocketAddr, @@ -237,7 +247,15 @@ impl Plan { err, } })? { - Err(PlanError::FoundV2) + Err(PlanError::FoundV3) + } else if Self::has_v4(storage_manager).await.map_err(|err| { + // Same as the comment above, but for version 4. + PlanError::Io { + message: String::from("looking for v4 RSS plan"), + err, + } + })? { + Err(PlanError::FoundV4) } else { Ok(None) } @@ -300,6 +318,25 @@ impl Plan { Ok(false) } + async fn has_v4( + storage_manager: &StorageHandle, + ) -> Result { + let paths = storage_manager + .get_latest_disks() + .await + .all_m2_mountpoints(CONFIG_DATASET) + .into_iter() + .map(|p| p.join(RSS_SERVICE_PLAN_V4_FILENAME)); + + for p in paths { + if p.try_exists()? { + return Ok(true); + } + } + + Ok(false) + } + async fn is_sled_scrimlet( log: &Logger, address: SocketAddrV6, @@ -465,7 +502,7 @@ impl Plan { .host_zone_with_one_backend( id, ServiceName::InternalDns, - dns_address, + http_address, ) .unwrap(); let dataset_name = @@ -1344,10 +1381,10 @@ mod tests { } #[test] - fn test_rss_service_plan_v4_schema() { + fn test_rss_service_plan_v5_schema() { let schema = schemars::schema_for!(Plan); expectorate::assert_contents( - "../schema/rss-service-plan-v4.json", + "../schema/rss-service-plan-v5.json", &serde_json::to_string_pretty(&schema).unwrap(), ); } diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index a5ba8d9d7f..2795422ee2 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -17,7 +17,7 @@ //! state files that get generated as RSS executes: //! //! - /pool/int/UUID/config/rss-sled-plan.json (Sled Plan) -//! - /pool/int/UUID/config/rss-service-plan-v3.json (Service Plan) +//! - /pool/int/UUID/config/rss-service-plan-v5.json (Service Plan) //! - /pool/int/UUID/config/rss-plan-completed.marker (Plan Execution Complete) //! //! These phases are described below. As each phase completes, a corresponding @@ -90,8 +90,9 @@ use nexus_sled_agent_shared::inventory::{ OmicronZoneConfig, OmicronZoneType, OmicronZonesConfig, }; use nexus_types::deployment::{ - blueprint_zone_type, Blueprint, BlueprintZoneType, BlueprintZonesConfig, - CockroachDbPreserveDowngrade, + blueprint_zone_type, Blueprint, BlueprintDatasetConfig, + BlueprintDatasetDisposition, BlueprintDatasetsConfig, BlueprintZoneType, + BlueprintZonesConfig, CockroachDbPreserveDowngrade, }; use nexus_types::external_api::views::SledState; use omicron_common::address::get_sled_address; @@ -843,6 +844,15 @@ impl ServiceInner { management_addrs: lp.management_addrs.clone(), } }), + tx_eq: config.tx_eq.as_ref().map(|tx_eq| { + NexusTypes::TxEqConfig { + pre1: tx_eq.pre1, + pre2: tx_eq.pre2, + main: tx_eq.main, + post2: tx_eq.post2, + post1: tx_eq.post1, + } + }), }) .collect(), bgp: config @@ -1442,6 +1452,47 @@ pub(crate) fn build_initial_blueprint_from_sled_configs( .map(|(sled_id, sled_config)| (*sled_id, sled_config.disks.clone())) .collect(); + let mut blueprint_datasets = BTreeMap::new(); + for (sled_id, sled_config) in sled_configs_by_id { + let mut datasets = BTreeMap::new(); + for d in sled_config.datasets.datasets.values() { + // Only the "Crucible" dataset needs to know the address + let address = sled_config.zones.iter().find_map(|z| { + if let BlueprintZoneType::Crucible( + blueprint_zone_type::Crucible { address, dataset }, + ) = &z.zone_type + { + if &dataset.pool_name == d.name.pool() { + return Some(*address); + } + }; + None + }); + + datasets.insert( + d.id, + BlueprintDatasetConfig { + disposition: BlueprintDatasetDisposition::InService, + id: d.id, + pool: d.name.pool().clone(), + kind: d.name.dataset().clone(), + address, + compression: d.compression, + quota: d.quota, + reservation: d.reservation, + }, + ); + } + + blueprint_datasets.insert( + *sled_id, + BlueprintDatasetsConfig { + generation: sled_config.datasets.generation, + datasets, + }, + ); + } + let mut blueprint_zones = BTreeMap::new(); let mut sled_state = BTreeMap::new(); for (sled_id, sled_config) in sled_configs_by_id { @@ -1468,6 +1519,7 @@ pub(crate) fn build_initial_blueprint_from_sled_configs( id: Uuid::new_v4(), blueprint_zones, blueprint_disks, + blueprint_datasets, sled_state, parent_blueprint_id: None, internal_dns_version, diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 0243060546..cf7ca60d05 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -89,7 +89,6 @@ use omicron_common::backoff::{ use omicron_common::disk::{DatasetKind, DatasetName}; use omicron_common::ledger::{self, Ledger, Ledgerable}; use omicron_ddm_admin_client::{Client as DdmAdminClient, DdmError}; -use omicron_uuid_kinds::GenericUuid; use once_cell::sync::OnceCell; use rand::prelude::SliceRandom; use sled_agent_types::{ @@ -1516,8 +1515,7 @@ impl ServiceManager { Some(dir) => ZoneBuilderFactory::fake(Some(dir)).builder(), }; if let Some(uuid) = unique_name { - zone_builder = - zone_builder.with_unique_name(uuid.into_untyped_uuid()); + zone_builder = zone_builder.with_unique_name(uuid); } if let Some(vnic) = bootstrap_vnic { zone_builder = zone_builder.with_bootstrap_vnic(vnic); @@ -3621,6 +3619,7 @@ impl ServiceManager { illumos_utils::zfs::Zfs::get_values( &dataset.full_name(), &["zoned", "canmount", "encryption"], + None, ) .map_err(|err| Error::GetZfsValue { zone: zone.zone_name(), diff --git a/sled-agent/src/sim/artifact_store.rs b/sled-agent/src/sim/artifact_store.rs new file mode 100644 index 0000000000..d73f5a2880 --- /dev/null +++ b/sled-agent/src/sim/artifact_store.rs @@ -0,0 +1,48 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Implementation of `crate::artifact_store::StorageBackend` for our simulated +//! storage. + +use std::sync::Arc; + +use camino_tempfile::Utf8TempDir; +use futures::lock::Mutex; +use sled_storage::error::Error as StorageError; + +use super::storage::Storage; +use crate::artifact_store::DatasetsManager; + +pub(super) struct SimArtifactStorage { + root: Utf8TempDir, + backend: Arc>, +} + +impl SimArtifactStorage { + pub(super) fn new(backend: Arc>) -> SimArtifactStorage { + SimArtifactStorage { + root: camino_tempfile::tempdir().unwrap(), + backend, + } + } +} + +impl DatasetsManager for SimArtifactStorage { + async fn artifact_storage_paths( + &self, + ) -> Result + '_, StorageError> + { + let config = self + .backend + .lock() + .await + .datasets_config_list() + .await + .map_err(|_| StorageError::LedgerNotFound)?; + Ok(crate::artifact_store::filter_dataset_mountpoints( + config, + self.root.path(), + )) + } +} diff --git a/sled-agent/src/sim/disk.rs b/sled-agent/src/sim/disk.rs index 9661b1949b..f08d010ae6 100644 --- a/sled-agent/src/sim/disk.rs +++ b/sled-agent/src/sim/disk.rs @@ -18,7 +18,6 @@ use omicron_common::api::internal::nexus::ProducerEndpoint; use omicron_common::api::internal::nexus::ProducerKind; use oximeter_producer::LogConfig; use oximeter_producer::Server as ProducerServer; -use propolis_client::types::DiskAttachmentState as PropolisDiskState; use sled_agent_types::disk::DiskStateRequested; use std::net::{Ipv6Addr, SocketAddr}; use std::sync::Arc; @@ -222,18 +221,13 @@ impl Simulatable for SimDisk { fn execute_desired_transition(&mut self) -> Option { if let Some(desired) = self.state.desired() { - // These operations would typically be triggered via responses from - // Propolis, but for a simulated sled agent, this does not exist. - // - // Instead, we make transitions to new states based entirely on the - // value of "desired". let observed = match desired { DiskStateRequested::Attached(uuid) => { - PropolisDiskState::Attached(*uuid) + DiskState::Attached(*uuid) } - DiskStateRequested::Detached => PropolisDiskState::Detached, - DiskStateRequested::Destroyed => PropolisDiskState::Destroyed, - DiskStateRequested::Faulted => PropolisDiskState::Faulted, + DiskStateRequested::Detached => DiskState::Detached, + DiskStateRequested::Destroyed => DiskState::Destroyed, + DiskStateRequested::Faulted => DiskState::Faulted, }; self.state.observe_transition(&observed) } else { diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index ca7f5e3410..fdffb249cf 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -10,6 +10,7 @@ use dropshot::endpoint; use dropshot::ApiDescription; use dropshot::FreeformBody; use dropshot::HttpError; +use dropshot::HttpResponseAccepted; use dropshot::HttpResponseCreated; use dropshot::HttpResponseDeleted; use dropshot::HttpResponseHeaders; @@ -35,6 +36,7 @@ use omicron_common::disk::DatasetsConfig; use omicron_common::disk::DatasetsManagementResult; use omicron_common::disk::DisksManagementResult; use omicron_common::disk::OmicronPhysicalDisksConfig; +use omicron_common::update::ArtifactHash; use sled_agent_api::*; use sled_agent_types::boot_disk::BootDiskOsWriteStatus; use sled_agent_types::boot_disk::BootDiskPathParams; @@ -73,7 +75,6 @@ pub fn api() -> SledApiDescription { api.register(instance_poke_single_step_post)?; api.register(instance_post_sim_migration_source)?; api.register(disk_poke_post)?; - Ok(api) } @@ -182,6 +183,48 @@ impl SledAgentApi for SledAgentSimImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn artifact_list( + rqctx: RequestContext, + ) -> Result>, HttpError> { + Ok(HttpResponseOk(rqctx.context().artifact_store().list().await?)) + } + + async fn artifact_copy_from_depot( + rqctx: RequestContext, + path_params: Path, + body: TypedBody, + ) -> Result, HttpError> + { + let sha256 = path_params.into_inner().sha256; + let depot_base_url = body.into_inner().depot_base_url; + rqctx + .context() + .artifact_store() + .copy_from_depot(sha256, &depot_base_url) + .await?; + Ok(HttpResponseAccepted(ArtifactCopyFromDepotResponse {})) + } + + async fn artifact_put( + rqctx: RequestContext, + path_params: Path, + body: StreamingBody, + ) -> Result, HttpError> { + let sha256 = path_params.into_inner().sha256; + Ok(HttpResponseOk( + rqctx.context().artifact_store().put_body(sha256, body).await?, + )) + } + + async fn artifact_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let sha256 = path_params.into_inner().sha256; + rqctx.context().artifact_store().delete(sha256).await?; + Ok(HttpResponseDeleted()) + } + async fn vmm_issue_disk_snapshot_request( rqctx: RequestContext, path_params: Path, diff --git a/sled-agent/src/sim/mod.rs b/sled-agent/src/sim/mod.rs index 14d980cf79..ab3b155b36 100644 --- a/sled-agent/src/sim/mod.rs +++ b/sled-agent/src/sim/mod.rs @@ -4,6 +4,7 @@ //! Simulated sled agent implementation +mod artifact_store; mod collection; mod config; mod disk; diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 7336182744..6301228efe 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -534,9 +534,15 @@ pub async fn run_standalone_server( None => vec![], }; - let disks = server.sled_agent.omicron_physical_disks_list().await?; let mut sled_configs = BTreeMap::new(); - sled_configs.insert(config.id, SledConfig { disks, zones }); + sled_configs.insert( + config.id, + SledConfig { + disks: server.sled_agent.omicron_physical_disks_list().await?, + datasets: server.sled_agent.datasets_config_list().await?, + zones, + }, + ); let rack_init_request = NexusTypes::RackInitializationRequest { blueprint: build_initial_blueprint_from_sled_configs( diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 321e9cc34f..4403116bab 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -4,12 +4,14 @@ //! Simulated sled agent implementation +use super::artifact_store::SimArtifactStorage; use super::collection::{PokeMode, SimCollection}; use super::config::Config; use super::disk::SimDisk; use super::instance::{self, SimInstance}; use super::storage::CrucibleData; use super::storage::Storage; +use crate::artifact_store::ArtifactStore; use crate::nexus::NexusClient; use crate::sim::simulatable::Simulatable; use crate::updates::UpdateManager; @@ -72,7 +74,7 @@ pub struct SledAgent { vmms: Arc>, /// collection of simulated disks, indexed by disk uuid disks: Arc>, - storage: Mutex, + storage: Arc>, updates: UpdateManager, nexus_address: SocketAddr, pub nexus_client: Arc, @@ -88,6 +90,7 @@ pub struct SledAgent { fake_zones: Mutex, instance_ensure_state_error: Mutex>, pub bootstore_network_config: Mutex, + artifacts: ArtifactStore, pub log: Logger, } @@ -165,6 +168,14 @@ impl SledAgent { }, }); + let storage = Arc::new(Mutex::new(Storage::new( + id.into_untyped_uuid(), + config.storage.ip, + storage_log, + ))); + let artifacts = + ArtifactStore::new(&log, SimArtifactStorage::new(storage.clone())); + Arc::new(SledAgent { id, ip: config.dropshot.bind_address.ip(), @@ -178,11 +189,7 @@ impl SledAgent { disk_log, sim_mode, )), - storage: Mutex::new(Storage::new( - id.into_untyped_uuid(), - config.storage.ip, - storage_log, - )), + storage, updates: UpdateManager::new(config.updates.clone()), nexus_address, nexus_client, @@ -197,6 +204,7 @@ impl SledAgent { zones: vec![], }), instance_ensure_state_error: Mutex::new(None), + artifacts, log, bootstore_network_config, }) @@ -338,14 +346,12 @@ impl SledAgent { id: propolis_id.into_untyped_uuid(), name: hardware.properties.hostname.to_string(), description: "sled-agent-sim created instance".to_string(), - image_id: Uuid::default(), - bootrom_id: Uuid::default(), - memory: hardware.properties.memory.to_whole_mebibytes(), - vcpus: hardware.properties.ncpus.0 as u8, metadata, }; let body = propolis_client::types::InstanceEnsureRequest { properties, + memory: hardware.properties.memory.to_whole_mebibytes(), + vcpus: hardware.properties.ncpus.0 as u8, nics: vec![], disks: vec![], boot_settings: None, @@ -560,6 +566,10 @@ impl SledAgent { &self.updates } + pub(super) fn artifact_store(&self) -> &ArtifactStore { + &self.artifacts + } + pub async fn vmm_count(&self) -> usize { self.vmms.size().await } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 8a5b15aaaf..3106c3bb38 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -4,6 +4,7 @@ //! Sled agent implementation +use crate::artifact_store::ArtifactStore; use crate::boot_disk_os_writer::BootDiskOsWriter; use crate::bootstrap::config::BOOTSTRAP_AGENT_RACK_INIT_PORT; use crate::bootstrap::early_networking::EarlyNetworkSetupError; @@ -167,6 +168,9 @@ pub enum Error { #[error("Expected revision to fit in a u32, but found {0}")] UnexpectedRevision(i64), + + #[error(transparent)] + RepoDepotStart(#[from] crate::artifact_store::StartError), } impl From for omicron_common::api::external::Error { @@ -360,6 +364,9 @@ struct SledAgentInner { // Component of Sled Agent responsible for managing instrumentation probes. probes: ProbeManager, + + // Component of Sled Agent responsible for managing the artifact store. + repo_depot: dropshot::HttpServer>, } impl SledAgentInner { @@ -592,6 +599,10 @@ impl SledAgent { log.new(o!("component" => "ProbeManager")), ); + let repo_depot = ArtifactStore::new(&log, storage_manager.clone()) + .start(sled_address, &config.dropshot) + .await?; + let sled_agent = SledAgent { inner: Arc::new(SledAgentInner { id: request.body.id, @@ -614,6 +625,7 @@ impl SledAgent { bootstore: long_running_task_handles.bootstore.clone(), _metrics_manager: metrics_manager, boot_disk_os_writer: BootDiskOsWriter::new(&parent_log), + repo_depot, }), log: log.clone(), sprockets: config.sprockets.clone(), @@ -1089,6 +1101,8 @@ impl SledAgent { } /// Downloads and applies an artifact. + // TODO: This is being split into "download" (i.e. store an artifact in the + // artifact store) and "apply" (perform an update using an artifact). pub async fn update_artifact( &self, artifact: UpdateArtifactId, @@ -1100,6 +1114,10 @@ impl SledAgent { Ok(()) } + pub fn artifact_store(&self) -> &ArtifactStore { + &self.inner.repo_depot.app_private() + } + /// Issue a snapshot request for a Crucible disk attached to an instance pub async fn vmm_issue_disk_snapshot_request( &self, diff --git a/sled-agent/tests/integration_tests/early_network.rs b/sled-agent/tests/integration_tests/early_network.rs index c0ecc09f12..420ce493d0 100644 --- a/sled-agent/tests/integration_tests/early_network.rs +++ b/sled-agent/tests/integration_tests/early_network.rs @@ -154,6 +154,7 @@ fn current_config_example() -> (&'static str, EarlyNetworkConfig) { vlan_id: None, }], autoneg: true, + tx_eq: None, lldp: None, }], bgp: vec![BgpConfig { diff --git a/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json b/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json index 270d926ea8..25e470e389 100644 --- a/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json +++ b/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json @@ -144,7 +144,8 @@ "uplink_port_fec": "none", "bgp_peers": [], "autoneg": false, - "lldp": null + "lldp": null, + "tx_eq": null }, { "routes": [ @@ -167,7 +168,8 @@ "uplink_port_fec": "none", "bgp_peers": [], "autoneg": false, - "lldp": null + "lldp": null, + "tx_eq": null } ], "bgp": [], diff --git a/sled-agent/types/src/early_networking.rs b/sled-agent/types/src/early_networking.rs index 46ceb2dbbf..6faead6fe9 100644 --- a/sled-agent/types/src/early_networking.rs +++ b/sled-agent/types/src/early_networking.rs @@ -300,6 +300,7 @@ pub mod back_compat { bgp_peers: v1.bgp_peers.clone(), autoneg: v1.autoneg, lldp: None, + tx_eq: None, } } } @@ -347,6 +348,7 @@ pub mod back_compat { bgp_peers: vec![], autoneg: false, lldp: None, + tx_eq: None, } } } @@ -520,6 +522,7 @@ mod tests { autoneg: false, bgp_peers: vec![], lldp: None, + tx_eq: None, }], bgp: vec![], bfd: vec![], @@ -602,6 +605,7 @@ mod tests { autoneg: false, bgp_peers: vec![], lldp: None, + tx_eq: None, }], bgp: vec![], bfd: vec![], diff --git a/sled-storage/src/dataset.rs b/sled-storage/src/dataset.rs index a715a33a69..0f2b610c42 100644 --- a/sled-storage/src/dataset.rs +++ b/sled-storage/src/dataset.rs @@ -30,6 +30,7 @@ pub const CLUSTER_DATASET: &'static str = "cluster"; pub const CONFIG_DATASET: &'static str = "config"; pub const M2_DEBUG_DATASET: &'static str = "debug"; pub const M2_BACKING_DATASET: &'static str = "backing"; +pub const M2_ARTIFACT_DATASET: &'static str = "update"; pub const DEBUG_DATASET_QUOTA: ByteCount = if cfg!(any(test, feature = "testing")) { @@ -46,6 +47,10 @@ pub const DUMP_DATASET_QUOTA: ByteCount = ByteCount::from_gibibytes_u32(100); // passed to zfs create -o compression= pub const DUMP_DATASET_COMPRESSION: CompressionAlgorithm = CompressionAlgorithm::GzipN { level: GzipLevel::new::<9>() }; +// TODO-correctness: This value of 20 GiB is a wild guess -- given TUF repo +// sizes as of Oct 2024, it would be capable of storing about 10 distinct system +// versions. +pub const ARTIFACT_DATASET_QUOTA: ByteCount = ByteCount::from_gibibytes_u32(20); // U.2 datasets live under the encrypted dataset and inherit encryption pub const ZONE_DATASET: &'static str = "crypt/zone"; @@ -65,7 +70,7 @@ const U2_EXPECTED_DATASETS: [ExpectedDataset; U2_EXPECTED_DATASET_COUNT] = [ .compression(DUMP_DATASET_COMPRESSION), ]; -const M2_EXPECTED_DATASET_COUNT: usize = 6; +const M2_EXPECTED_DATASET_COUNT: usize = 7; const M2_EXPECTED_DATASETS: [ExpectedDataset; M2_EXPECTED_DATASET_COUNT] = [ // Stores software images. // @@ -90,6 +95,9 @@ const M2_EXPECTED_DATASETS: [ExpectedDataset; M2_EXPECTED_DATASET_COUNT] = [ ExpectedDataset::new(CONFIG_DATASET), // Store debugging data, such as service bundles. ExpectedDataset::new(M2_DEBUG_DATASET).quota(DEBUG_DATASET_QUOTA), + // Stores software artifacts (zones, OS images, Hubris images, etc.) + // extracted from TUF repos by Nexus. + ExpectedDataset::new(M2_ARTIFACT_DATASET).quota(ARTIFACT_DATASET_QUOTA), ]; // Helper type for describing expected datasets and their optional quota. diff --git a/smf/clickhouse/config.xml b/smf/clickhouse/config.xml index b5b1f9c17f..58ae5dcaf5 100644 --- a/smf/clickhouse/config.xml +++ b/smf/clickhouse/config.xml @@ -38,4 +38,8 @@ + + + 1.0 + diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index eff73799ee..a7236f5142 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -43,6 +43,7 @@ uuid.workspace = true chrono.workspace = true expectorate.workspace = true gethostname.workspace = true +serde.workspace = true [features] seed-gen = ["dep:filetime"] diff --git a/test-utils/src/dev/clickhouse.rs b/test-utils/src/dev/clickhouse.rs index 8aa7b319c4..8956bb88a7 100644 --- a/test-utils/src/dev/clickhouse.rs +++ b/test-utils/src/dev/clickhouse.rs @@ -387,6 +387,9 @@ pub enum ClickHouseError { Timeout, } +const SINGLE_NODE_CONFIG_FILE: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/../smf/clickhouse/config.xml"); + impl ClickHouseProcess { /// Start a new single node ClickHouse server listening on the provided /// ports. @@ -395,8 +398,22 @@ impl ClickHouseProcess { ports: ClickHousePorts, ) -> Result { let data_dir = ClickHouseDataDir::new(logctx)?; + + // Copy the configuration file into the test directory, so we don't + // leave the preprocessed config file hanging around. + tokio::fs::copy(SINGLE_NODE_CONFIG_FILE, data_dir.config_file_path()) + .await + .with_context(|| { + format!( + "failed to copy config file from {} to test data path {}", + SINGLE_NODE_CONFIG_FILE, + data_dir.config_file_path(), + ) + })?; let args = vec![ "server".to_string(), + "--config-file".to_string(), + data_dir.config_file_path().to_string(), "--log-file".to_string(), data_dir.log_path().to_string(), "--errorlog-file".to_string(), @@ -731,6 +748,10 @@ impl ClickHouseDataDir { self.dir.path() } + fn config_file_path(&self) -> Utf8PathBuf { + self.root_path().join("config.xml") + } + fn datastore_path(&self) -> Utf8PathBuf { self.dir.path().join("datastore/") } @@ -1299,10 +1320,11 @@ async fn clickhouse_ready_from_log( #[cfg(test)] mod tests { use super::{ - discover_ready, wait_for_ports, ClickHouseError, ClickHousePorts, - CLICKHOUSE_HTTP_PORT_NEEDLE, CLICKHOUSE_READY, + discover_ready, wait_for_ports, ClickHouseDeployment, ClickHouseError, + ClickHousePorts, CLICKHOUSE_HTTP_PORT_NEEDLE, CLICKHOUSE_READY, CLICKHOUSE_TCP_PORT_NEEDLE, CLICKHOUSE_TIMEOUT, }; + use crate::dev::test_setup_log; use camino_tempfile::NamedUtf8TempFile; use std::process::Stdio; use std::{io::Write, sync::Arc, time::Duration}; @@ -1458,4 +1480,39 @@ mod tests { let second = ClickHousePorts { http: 1, native: 1 }; ClickHousePorts { http: 2, native: 2 }.assert_consistent(&second); } + + #[derive(Debug, serde::Deserialize)] + struct Setting { + name: String, + value: String, + changed: u8, + } + + #[tokio::test] + async fn sparse_serialization_is_disabled() { + let logctx = test_setup_log("sparse_serialization_is_disabled"); + let mut db = + ClickHouseDeployment::new_single_node(&logctx).await.unwrap(); + let client = reqwest::Client::new(); + let url = format!("http://{}", db.http_address()); + let setting: Setting = client + .post(url) + .body( + "SELECT name, value, changed \ + FROM system.merge_tree_settings \ + WHERE name == 'ratio_of_defaults_for_sparse_serialization' \ + FORMAT JSONEachRow", + ) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(setting.name, "ratio_of_defaults_for_sparse_serialization"); + assert_eq!(setting.value, "1"); + assert_eq!(setting.changed, 1); + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } } diff --git a/tools/console_version b/tools/console_version index 81623a50f8..4b01d544d1 100644 --- a/tools/console_version +++ b/tools/console_version @@ -1,2 +1,2 @@ -COMMIT="f7d6dafd110a1c3489d87ebf6fccf7175e895009" -SHA2="7b9def2b20871a362d1797ca84c072d5c85727b0cc04b66909f7438e87934181" +COMMIT="6eeab2037c3440126f0df06a843127890083bd93" +SHA2="afd6078e39d7c76d86e8b9093afb9d9fcdf8a7290091c7b32b4cfc6db0a8b4f9" diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index 75c57e9d29..09378177d7 100755 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="f3810e7bc1f0d746b5e95b3aaff32e52b02dfdfa" -SHA2="3a54305ab4b1270c9a5fb0603f481fce199f3767c174a03559ff642f7f44687e" +COMMIT="4067d742d832fa434217b95e4b149048d01ef54e" +SHA2="ff41c2a30f67c4ce79fc1c8dd99bce8042e855c51fd15362be4ee833acec58cf" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index fe420e299e..40cf08e5c0 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="c1506f6f818327523e6ff3102432a2038d319338b883235664b34f9132ff676a" -CIDL_SHA256_LINUX_DPD="fc9ea4dc22e761dce3aa4d252983360f799426a0c23ea8f347653664d3e2b55a" -CIDL_SHA256_LINUX_SWADM="9da0dd6c972206338971a90144b1c35e101d69aaacf26240a45cef45d828b090" +CIDL_SHA256_ILLUMOS="5e9ccc42e5ac31f4be24025d2afd5978aef33d618f3cb7caa260eff73b7e6a79" +CIDL_SHA256_LINUX_DPD="5738cb74ea2657dd255f5ba094600265f5b10b99eeb70bd0b69cbb987b951b71" +CIDL_SHA256_LINUX_SWADM="30e085b64e33c374ec79d9e0e8cf015547e573b34525d2da228a8c99755557fb" diff --git a/tools/permslip_production b/tools/permslip_production index b985e1280c..a01da94672 100644 --- a/tools/permslip_production +++ b/tools/permslip_production @@ -1,2 +1,2 @@ -926c726edca29f7134df9489fdedbdc3553d7e9ecbaf97d3be6c55dc2634f828 manifest-oxide-rot-1-v1.0.28.toml +3cbe182c3e3425442eec2c2019045f8d5e613688597a01c6f05fa93e2ff3d165 manifest-oxide-rot-1-v1.0.27.toml 610ebce44b1fb622eb56591534fb2569340fdba9b5ba62ca1b02f0b2d2e973dc manifest-bootleby-v1.3.1.toml diff --git a/tools/permslip_staging b/tools/permslip_staging index 62cf33bace..2882d76c6d 100644 --- a/tools/permslip_staging +++ b/tools/permslip_staging @@ -1,5 +1,5 @@ -ffb2be39e9bd1c5f2203d414c63e86afb8f820a8938119d7c6b29f9a8aa42c29 manifest-gimlet-v1.0.30 +3850f287e8f16d4b84ed157761487b64a7cea36c9a52ef5bb22f8cbbe022e1cb manifest-gimlet-v1.0.31.toml 82f68363c5f89acb8f1e9f0fdec00694d84bd69291a63f9c3c3141721a42af9a manifest-oxide-rot-1-v1.0.28.toml -ce877624a26ac5b2a122816eda89316ea98c101cbfd61fbb4c393188a963c3de manifest-psc-v1.0.29.toml -81e9889178ce147d9aef6925c416e1e75c1b38e29a4daed7f7f287f86a9360b7 manifest-sidecar-v1.0.29.toml +ae9ddf5f9a16a780dd25aa6259f0492ce74a792e2802c8c7c3ef9163e556fb15 manifest-psc-v1.0.30.toml +bea28961a57d2cf3d123df05a1eb9ec87fdde2862b197416b0f2531b6db4f06e manifest-sidecar-v1.0.30.toml 6f8459afe22c27d5920356878e4d8d639464f39a15ce7b5b040c2d908d52a570 manifest-bootleby-v1.3.1.toml diff --git a/tufaceous-lib/src/archive.rs b/tufaceous-lib/src/archive.rs index 9440db5a4c..675cba7eee 100644 --- a/tufaceous-lib/src/archive.rs +++ b/tufaceous-lib/src/archive.rs @@ -14,7 +14,10 @@ use std::{ fmt, io::{BufReader, BufWriter, Cursor, Read, Seek}, }; -use zip::{write::FileOptions, CompressionMethod, ZipArchive, ZipWriter}; +use zip::{ + write::{FileOptions, SimpleFileOptions}, + CompressionMethod, ZipArchive, ZipWriter, +}; /// A builder for TUF repo archives. #[derive(Debug)] @@ -65,18 +68,20 @@ impl ArchiveBuilder { Ok(()) } - pub fn finish(mut self) -> Result<()> { - let zip_file = self.writer.finish().with_context(|| { - format!("error finalizing archive at `{}`", self.output_path) + pub fn finish(self) -> Result<()> { + let Self { writer, output_path } = self; + + let zip_file = writer.0.finish().with_context(|| { + format!("error finalizing archive at `{}`", output_path) })?; zip_file.into_inner().with_context(|| { - format!("error writing archive at `{}`", self.output_path) + format!("error writing archive at `{}`", output_path) })?; Ok(()) } - fn file_options() -> FileOptions { + fn file_options() -> SimpleFileOptions { // The main purpose of the zip archive is to transmit archives that are // already compressed, so there's no point trying to re-compress them. FileOptions::default().compression_method(CompressionMethod::Stored) diff --git a/uuid-kinds/src/lib.rs b/uuid-kinds/src/lib.rs index ba586c03a5..7947062a82 100644 --- a/uuid-kinds/src/lib.rs +++ b/uuid-kinds/src/lib.rs @@ -49,6 +49,7 @@ macro_rules! impl_typed_uuid_kind { // Please keep this list in alphabetical order. impl_typed_uuid_kind! { + Blueprint => "blueprint", Collection => "collection", Dataset => "dataset", DemoSaga => "demo_saga", diff --git a/wicket-common/src/example.rs b/wicket-common/src/example.rs index 3951520f01..8572b791cd 100644 --- a/wicket-common/src/example.rs +++ b/wicket-common/src/example.rs @@ -13,7 +13,7 @@ use omicron_common::{ external::AllowedSourceIps, internal::shared::{ BgpConfig, BgpPeerConfig, LldpAdminStatus, LldpPortConfig, PortFec, - PortSpeed, RouteConfig, + PortSpeed, RouteConfig, TxEqConfig, }, }, }; @@ -177,6 +177,14 @@ impl ExampleRackSetupData { management_addrs: None, }); + let tx_eq = Some(TxEqConfig { + pre1: Some(0), + pre2: Some(0), + main: Some(26), + post2: Some(0), + post1: Some(0), + }); + let switch1_port0_lldp = Some(LldpPortConfig { status: LldpAdminStatus::Enabled, chassis_id: Some("chassid id override".to_string()), @@ -190,10 +198,11 @@ impl ExampleRackSetupData { let rack_network_config = UserSpecifiedRackNetworkConfig { infra_ip_first: "172.30.0.1".parse().unwrap(), infra_ip_last: "172.30.0.10".parse().unwrap(), + #[rustfmt::skip] switch0: btreemap! { - "port0".to_owned() => UserSpecifiedPortConfig { - addresses: vec!["172.30.0.1/24".parse().unwrap()], - routes: vec![RouteConfig { + "port0".to_owned() => UserSpecifiedPortConfig { + addresses: vec!["172.30.0.1/24".parse().unwrap()], + routes: vec![RouteConfig { destination: "0.0.0.0/0".parse().unwrap(), nexthop: "172.30.0.10".parse().unwrap(), vlan_id: Some(1), @@ -202,10 +211,12 @@ impl ExampleRackSetupData { bgp_peers: switch0_port0_bgp_peers, uplink_port_speed: PortSpeed::Speed400G, uplink_port_fec: PortFec::Firecode, - lldp: switch0_port0_lldp, - autoneg: true, - }, - }, + lldp: switch0_port0_lldp, + tx_eq, + autoneg: true, + }, + }, + #[rustfmt::skip] switch1: btreemap! { // Use the same port name as in switch0 to test that it doesn't // collide. @@ -221,6 +232,7 @@ impl ExampleRackSetupData { uplink_port_speed: PortSpeed::Speed400G, uplink_port_fec: PortFec::Firecode, lldp: switch1_port0_lldp, + tx_eq, autoneg: true, }, }, diff --git a/wicket-common/src/rack_setup.rs b/wicket-common/src/rack_setup.rs index cb6b13422b..7136bc8d96 100644 --- a/wicket-common/src/rack_setup.rs +++ b/wicket-common/src/rack_setup.rs @@ -15,6 +15,7 @@ use omicron_common::api::internal::shared::LldpPortConfig; use omicron_common::api::internal::shared::PortFec; use omicron_common::api::internal::shared::PortSpeed; use omicron_common::api::internal::shared::RouteConfig; +use omicron_common::api::internal::shared::TxEqConfig; use omicron_common::api::internal::shared::UplinkAddressConfig; use omicron_common::update::ArtifactHash; use owo_colors::OwoColorize; @@ -188,6 +189,8 @@ pub struct UserSpecifiedPortConfig { pub bgp_peers: Vec, #[serde(default)] pub lldp: Option, + #[serde(default)] + pub tx_eq: Option, } /// User-specified version of [`BgpPeerConfig`]. diff --git a/wicket/src/cli/rack_setup/config_toml.rs b/wicket/src/cli/rack_setup/config_toml.rs index 92496e94d5..727f1bbaa8 100644 --- a/wicket/src/cli/rack_setup/config_toml.rs +++ b/wicket/src/cli/rack_setup/config_toml.rs @@ -322,6 +322,7 @@ fn populate_uplink_table(cfg: &UserSpecifiedPortConfig) -> Table { autoneg, bgp_peers, lldp, + tx_eq, } = cfg; let mut uplink = Table::new(); @@ -533,6 +534,26 @@ fn populate_uplink_table(cfg: &UserSpecifiedPortConfig) -> Table { uplink.insert("lldp", Item::Table(lldp)); } + if let Some(t) = tx_eq { + let mut tx_eq = Table::new(); + if let Some(x) = t.pre1 { + tx_eq.insert("pre1", i32_item(x)); + } + if let Some(x) = t.pre2 { + tx_eq.insert("pre2", i32_item(x)); + } + if let Some(x) = t.main { + tx_eq.insert("main", i32_item(x)); + } + if let Some(x) = t.post1 { + tx_eq.insert("post1", i32_item(x)); + } + if let Some(x) = t.post2 { + tx_eq.insert("post2", i32_item(x)); + } + uplink.insert("tx_eq", Item::Table(tx_eq)); + } + uplink } @@ -556,6 +577,14 @@ fn string_item(s: impl ToString) -> Item { Item::Value(string_value(s)) } +fn i32_value(i: i32) -> Value { + Value::Integer(Formatted::new(i.into())) +} + +fn i32_item(i: i32) -> Item { + Item::Value(i32_value(i)) +} + fn i64_value(i: i64) -> Value { Value::Integer(Formatted::new(i)) } diff --git a/wicket/src/ui/panes/rack_setup.rs b/wicket/src/ui/panes/rack_setup.rs index be299ef022..1828049fc3 100644 --- a/wicket/src/ui/panes/rack_setup.rs +++ b/wicket/src/ui/panes/rack_setup.rs @@ -774,6 +774,7 @@ fn rss_config_text<'a>( autoneg, bgp_peers, lldp, + tx_eq, } = uplink; let mut items = vec![ @@ -1136,6 +1137,45 @@ fn rss_config_text<'a>( items.extend(lldp); } + if let Some(t) = tx_eq { + let mut tx_eq = vec![vec![Span::styled( + " • TxEq port settings: ", + label_style, + )]]; + + if let Some(x) = t.pre1 { + tx_eq.push(vec![ + Span::styled(" • Precursor 1: ", label_style), + Span::styled(x.to_string(), ok_style), + ]) + } + if let Some(x) = t.pre2 { + tx_eq.push(vec![ + Span::styled(" • Precursor 2: ", label_style), + Span::styled(x.to_string(), ok_style), + ]) + } + if let Some(x) = t.main { + tx_eq.push(vec![ + Span::styled(" • Main cursor: ", label_style), + Span::styled(x.to_string(), ok_style), + ]) + } + if let Some(x) = t.post2 { + tx_eq.push(vec![ + Span::styled(" • Postcursor 2: ", label_style), + Span::styled(x.to_string(), ok_style), + ]) + } + if let Some(x) = t.post1 { + tx_eq.push(vec![ + Span::styled(" • Postcursor 1: ", label_style), + Span::styled(x.to_string(), ok_style), + ]) + } + items.extend(tx_eq); + } + append_list( &mut spans, Cow::from(format!("Uplink {}: ", i + 1)), diff --git a/wicket/tests/output/example_non_empty.toml b/wicket/tests/output/example_non_empty.toml index fafb31048d..d6f5e7c820 100644 --- a/wicket/tests/output/example_non_empty.toml +++ b/wicket/tests/output/example_non_empty.toml @@ -119,6 +119,13 @@ system_name = "system name override" system_description = "system description override" port_description = "port description override" +[rack_network_config.switch0.port0.tx_eq] +pre1 = 0 +pre2 = 0 +main = 26 +post1 = 0 +post2 = 0 + [rack_network_config.switch1.port0] routes = [{ nexthop = "172.33.0.10", destination = "0.0.0.0/0", vlan_id = 1 }] addresses = [{ address = "172.32.0.1/24" }] @@ -148,6 +155,13 @@ system_description = "system description override" port_description = "port description override" management_addrs = ["172.32.0.4"] +[rack_network_config.switch1.port0.tx_eq] +pre1 = 0 +pre2 = 0 +main = 26 +post1 = 0 +post2 = 0 + [[rack_network_config.bgp]] asn = 47 originate = ["10.0.0.0/16"] diff --git a/wicketd/src/preflight_check/uplink.rs b/wicketd/src/preflight_check/uplink.rs index da30cd0199..e70d9db57b 100644 --- a/wicketd/src/preflight_check/uplink.rs +++ b/wicketd/src/preflight_check/uplink.rs @@ -778,6 +778,7 @@ fn build_port_settings( fec, speed, lane: Some(LinkId(0)), + tx_eq: None, }, }, ); diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index 2be098dbc2..8ee252ef77 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -702,6 +702,7 @@ fn build_port_config( use bootstrap_agent_client::types::PortSpeed as BaPortSpeed; use bootstrap_agent_client::types::RouteConfig as BaRouteConfig; use bootstrap_agent_client::types::SwitchLocation as BaSwitchLocation; + use bootstrap_agent_client::types::TxEqConfig as BaTxEqConfig; use bootstrap_agent_client::types::UplinkAddressConfig as BaUplinkAddressConfig; use omicron_common::api::internal::shared::LldpAdminStatus; use omicron_common::api::internal::shared::PortFec; @@ -807,6 +808,13 @@ fn build_port_config( port_description: c.port_description.clone(), management_addrs: c.management_addrs.clone(), }), + tx_eq: config.tx_eq.as_ref().map(|c| BaTxEqConfig { + pre1: c.pre1, + pre2: c.pre2, + main: c.main, + post2: c.post2, + post1: c.post1, + }), } } diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 9f6c51e0c0..c893112051 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -5,6 +5,7 @@ [package] name = "omicron-workspace-hack" version = "0.1.0" +edition = "2021" description = "workspace-hack package, managed by hakari" # You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing. publish = false @@ -19,7 +20,7 @@ workspace = true [dependencies] ahash = { version = "0.8.11" } aho-corasick = { version = "1.1.3" } -anyhow = { version = "1.0.89", features = ["backtrace"] } +anyhow = { version = "1.0.92", features = ["backtrace"] } base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } base64 = { version = "0.22.1" } base64ct = { version = "1.6.0", default-features = false, features = ["std"] } @@ -90,7 +91,7 @@ pkcs8 = { version = "0.10.2", default-features = false, features = ["encryption" postgres-types = { version = "0.2.8", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } predicates = { version = "3.1.2" } proc-macro2 = { version = "1.0.87" } -qorb = { version = "0.1.2", features = ["qtop"] } +qorb = { version = "0.2.0", features = ["qtop"] } quote = { version = "1.0.37" } rand = { version = "0.8.5", features = ["small_rng"] } regex = { version = "1.11.0" } @@ -130,11 +131,12 @@ uuid = { version = "1.10.0", features = ["serde", "v4"] } x509-cert = { version = "0.2.5" } zerocopy = { version = "0.7.35", features = ["derive", "simd"] } zeroize = { version = "1.8.1", features = ["std", "zeroize_derive"] } +zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [build-dependencies] ahash = { version = "0.8.11" } aho-corasick = { version = "1.1.3" } -anyhow = { version = "1.0.89", features = ["backtrace"] } +anyhow = { version = "1.0.92", features = ["backtrace"] } base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } base64 = { version = "0.22.1" } base64ct = { version = "1.6.0", default-features = false, features = ["std"] } @@ -206,7 +208,7 @@ pkcs8 = { version = "0.10.2", default-features = false, features = ["encryption" postgres-types = { version = "0.2.8", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } predicates = { version = "3.1.2" } proc-macro2 = { version = "1.0.87" } -qorb = { version = "0.1.2", features = ["qtop"] } +qorb = { version = "0.2.0", features = ["qtop"] } quote = { version = "1.0.37" } rand = { version = "0.8.5", features = ["small_rng"] } regex = { version = "1.11.0" } @@ -249,6 +251,7 @@ uuid = { version = "1.10.0", features = ["serde", "v4"] } x509-cert = { version = "0.2.5" } zerocopy = { version = "0.7.35", features = ["derive", "simd"] } zeroize = { version = "1.8.1", features = ["std", "zeroize_derive"] } +zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } [target.x86_64-unknown-linux-gnu.dependencies] cookie = { version = "0.18.1", default-features = false, features = ["percent-encode"] }