diff --git a/.config/nextest.toml b/.config/nextest.toml index 95d4c20102..eb62e1c9af 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -3,14 +3,16 @@ # # The required version should be bumped up if we need new features, performance # improvements or bugfixes that are present in newer versions of nextest. -nextest-version = { required = "0.9.64", recommended = "0.9.70" } +nextest-version = { required = "0.9.77", recommended = "0.9.77" } experimental = ["setup-scripts"] [[profile.default.scripts]] # Exclude omicron-dev tests from crdb-seed as we explicitly want to simulate an # environment where the seed file doesn't exist. -filter = 'rdeps(nexus-test-utils) - package(omicron-dev)' +# Exclude omicron-live-tests because those don't need this and also don't have +# it available in the environment in which they run. +filter = 'rdeps(nexus-test-utils) - package(omicron-dev) - package(omicron-live-tests)' setup = 'crdb-seed' [profile.ci] @@ -21,13 +23,20 @@ fail-fast = false # invocations of nextest happen. command = 'cargo run -p crdb-seed --profile test' +[test-groups] # The ClickHouse cluster tests currently rely on a hard-coded set of ports for # the nodes in the cluster. We would like to relax this in the future, at which # point this test-group configuration can be removed or at least loosened to # support testing in parallel. For now, enforce strict serialization for all # tests with `replicated` in the name. -[test-groups] clickhouse-cluster = { max-threads = 1 } +# While most Omicron tests operate with their own simulated control plane, the +# live-tests operate on a more realistic, shared control plane and test +# behaviors that conflict with each other. They need to be run serially. +live-tests = { max-threads = 1 } + +[profile.default] +default-filter = 'all() - package(omicron-live-tests) - package(end-to-end-tests)' [[profile.default.overrides]] filter = 'package(oximeter-db) and test(replicated)' @@ -43,3 +52,10 @@ filter = 'binary_id(omicron-nexus::test_all)' # As of 2023-01-08, the slowest test in test_all takes 196s on a Ryzen 7950X. # 900s is a good upper limit that adds a comfortable buffer. slow-timeout = { period = '60s', terminate-after = 15 } + +[profile.live-tests] +default-filter = 'package(omicron-live-tests)' + +[[profile.live-tests.overrides]] +filter = 'package(omicron-live-tests)' +test-group = 'live-tests' diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..ddfd1b04b7 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Whitespace-only changes +d01ba56c2127789d85723793380a7378394583f1 diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index 1e4b655cb9..1980012664 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -9,7 +9,7 @@ target_os=$1 # NOTE: This version should be in sync with the recommended version in # .config/nextest.toml. (Maybe build an automated way to pull the recommended # version in the future.) -NEXTEST_VERSION='0.9.70' +NEXTEST_VERSION='0.9.77' cargo --version rustc --version @@ -89,6 +89,12 @@ ptime -m timeout 2h cargo nextest run --profile ci --locked --verbose banner doctest ptime -m timeout 1h cargo test --doc --locked --verbose --no-fail-fast +# Build the live-tests. This is only supported on illumos. +# We also can't actually run them here. See the README for more details. +if [[ $target_os == "illumos" ]]; then + ptime -m cargo xtask live-tests +fi + # We expect the seed CRDB to be placed here, so we explicitly remove it so the # rmdir check below doesn't get triggered. Nextest doesn't have support for # teardown scripts so this is the best we've got. diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index 63752880d6..11aa638ace 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -24,7 +24,7 @@ jobs: with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@37129d5de13e9122cce55a7a5e7e49981cef514c # v2 + uses: taiki-e/install-action@11053896c3ed8d313b47efa789def6474abd1e6b # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date diff --git a/Cargo.lock b/Cargo.lock index 911386eb1b..0286e6b121 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,12 +256,14 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-bb8-diesel" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=ed7ab5ef0513ba303d33efd41d3e9e381169d59b#ed7ab5ef0513ba303d33efd41d3e9e381169d59b" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc03a2806f66f36513d65e0a7f34200382230250cadcf8a8397cfbe3f26b795" dependencies = [ "async-trait", "bb8", "diesel", + "futures", "thiserror", "tokio", ] @@ -703,7 +705,7 @@ dependencies = [ name = "bootstrap-agent-api" version = "0.1.0" dependencies = [ - "dropshot", + "dropshot 0.10.2-dev", "nexus-client", "omicron-common", "omicron-uuid-kinds", @@ -973,7 +975,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", - "dropshot", + "dropshot 0.10.2-dev", "futures", "libc", "omicron-rpaths", @@ -1117,7 +1119,7 @@ checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" name = "clickhouse-admin-api" version = "0.1.0" dependencies = [ - "dropshot", + "dropshot 0.10.2-dev", "omicron-common", "omicron-uuid-kinds", "omicron-workspace-hack", @@ -1125,6 +1127,22 @@ dependencies = [ "serde", ] +[[package]] +name = "clickhouse-admin-types" +version = "0.1.0" +dependencies = [ + "anyhow", + "camino", + "camino-tempfile", + "derive_more", + "expectorate", + "omicron-common", + "omicron-workspace-hack", + "schemars", + "serde", + "serde_json", +] + [[package]] name = "clickward" version = "0.1.0" @@ -1160,7 +1178,7 @@ name = "cockroach-admin-api" version = "0.1.0" dependencies = [ "cockroach-admin-types", - "dropshot", + "dropshot 0.10.2-dev", "omicron-common", "omicron-uuid-kinds", "omicron-workspace-hack", @@ -1387,7 +1405,7 @@ name = "crdb-seed" version = "0.1.0" dependencies = [ "anyhow", - "dropshot", + "dropshot 0.10.2-dev", "omicron-test-utils", "omicron-workspace-hack", "slog", @@ -1539,7 +1557,7 @@ dependencies = [ "anyhow", "atty", "crucible-workspace-hack", - "dropshot", + "dropshot 0.10.2-dev", "nix 0.28.0", "rusqlite", "rustls-pemfile 1.0.4", @@ -1925,6 +1943,15 @@ dependencies = [ "syn 2.0.74", ] +[[package]] +name = "des" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +dependencies = [ + "cipher", +] + [[package]] name = "dhcproto" version = "0.12.0" @@ -1948,9 +1975,9 @@ checksum = "a7993efb860416547839c115490d4951c6d0f8ec04a3594d9dd99d50ed7ec170" [[package]] name = "diesel" -version = "2.2.2" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf97ee7261bb708fa3402fa9c17a54b70e90e3cb98afb3dc8999d5512cb03f94" +checksum = "65e13bab2796f412722112327f3e575601a3e9cdcbe426f0d30dbf43f3f5dc71" dependencies = [ "bitflags 2.6.0", "byteorder", @@ -2023,15 +2050,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys", -] - [[package]] name = "dirs-next" version = "2.0.0" @@ -2042,18 +2060,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -2067,9 +2073,9 @@ dependencies = [ [[package]] name = "display-error-chain" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f77af9e75578c1ab34f5f04545a8b05be0c36fbd7a9bb3cf2d2a971e435fdbb9" +checksum = "7d305e5a3904ee14166439a70feef04853c1234226dbb27ede127b88dc5a4a9d" [[package]] name = "dladm" @@ -2104,7 +2110,7 @@ dependencies = [ "clap", "dns-server-api", "dns-service-client", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "hickory-client", "hickory-proto", @@ -2137,7 +2143,7 @@ name = "dns-server-api" version = "0.1.0" dependencies = [ "chrono", - "dropshot", + "dropshot 0.10.2-dev", "omicron-workspace-hack", "schemars", "serde", @@ -2210,6 +2216,52 @@ dependencies = [ "uuid", ] +[[package]] +name = "dropshot" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a391eeedf8a75a188eb670327c704b7ab10eb2bb890e2ec0880dd21d609fb6e8" +dependencies = [ + "async-stream", + "async-trait", + "base64 0.22.1", + "bytes", + "camino", + "chrono", + "debug-ignore", + "dropshot_endpoint 0.10.1", + "form_urlencoded", + "futures", + "hostname 0.4.0", + "http 0.2.12", + "hyper 0.14.30", + "indexmap 2.4.0", + "multer", + "openapiv3", + "paste", + "percent-encoding", + "rustls 0.22.4", + "rustls-pemfile 2.1.3", + "schemars", + "scopeguard", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", + "slog", + "slog-async", + "slog-bunyan", + "slog-json", + "slog-term", + "tokio", + "tokio-rustls 0.25.0", + "toml 0.8.19", + "uuid", + "version_check", + "waitgroup", +] + [[package]] name = "dropshot" version = "0.10.2-dev" @@ -2222,7 +2274,7 @@ dependencies = [ "camino", "chrono", "debug-ignore", - "dropshot_endpoint", + "dropshot_endpoint 0.10.2-dev", "form_urlencoded", "futures", "hostname 0.4.0", @@ -2256,6 +2308,19 @@ dependencies = [ "waitgroup", ] +[[package]] +name = "dropshot_endpoint" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9058c9c7e4a6b378cd12e71dc155bb15d0d4f8e1e6039ce2cf0a7c0c81043e33" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_tokenstream", + "syn 2.0.74", +] + [[package]] name = "dropshot_endpoint" version = "0.10.2-dev" @@ -2875,7 +2940,7 @@ dependencies = [ name = "gateway-api" version = "0.1.0" dependencies = [ - "dropshot", + "dropshot 0.10.2-dev", "gateway-types", "omicron-common", "omicron-uuid-kinds", @@ -2979,7 +3044,7 @@ name = "gateway-test-utils" version = "0.1.0" dependencies = [ "camino", - "dropshot", + "dropshot 0.10.2-dev", "gateway-messages", "gateway-types", "omicron-gateway", @@ -3988,7 +4053,7 @@ name = "installinator-api" version = "0.1.0" dependencies = [ "anyhow", - "dropshot", + "dropshot 0.10.2-dev", "hyper 0.14.30", "installinator-common", "omicron-common", @@ -4056,7 +4121,7 @@ dependencies = [ "chrono", "dns-server", "dns-service-client", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "futures", "hickory-resolver", @@ -4083,7 +4148,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", - "dropshot", + "dropshot 0.10.2-dev", "hickory-resolver", "internal-dns", "omicron-common", @@ -4102,9 +4167,8 @@ checksum = "fc6d6206008e25125b1f97fbe5d309eb7b85141cf9199d52dbd3729a1584dd16" name = "ipcc" version = "0.1.0" dependencies = [ - "cfg-if", "ciborium", - "libc", + "libipcc", "omicron-common", "omicron-workspace-hack", "proptest", @@ -4313,9 +4377,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.156" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5f43f184355eefb8d17fc948dbecf6c13be3c141f20d834ae842193a448c72a" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libdlpi-sys" @@ -4386,6 +4450,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libipcc" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/libipcc?rev=fdffa212373a8f92473ea5f411088912bf458d5f#fdffa212373a8f92473ea5f411088912bf458d5f" +dependencies = [ + "cfg-if", + "libc", + "thiserror", +] + [[package]] name = "libloading" version = "0.8.3" @@ -4541,6 +4615,15 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "live-tests-macros" +version = "0.1.0" +dependencies = [ + "omicron-workspace-hack", + "quote", + "syn 2.0.74", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -4918,7 +5001,7 @@ dependencies = [ "base64 0.22.1", "chrono", "cookie 0.18.1", - "dropshot", + "dropshot 0.10.2-dev", "futures", "headers", "http 0.2.12", @@ -4975,7 +5058,7 @@ version = "0.1.0" dependencies = [ "anyhow", "camino", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "libc", "omicron-common", @@ -5058,7 +5141,6 @@ dependencies = [ "assert_matches", "async-bb8-diesel", "async-trait", - "bb8", "camino", "camino-tempfile", "chrono", @@ -5066,7 +5148,7 @@ dependencies = [ "db-macros", "diesel", "diesel-dtrace", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "futures", "gateway-client", @@ -5103,6 +5185,7 @@ dependencies = [ "pq-sys", "predicates", "pretty_assertions", + "qorb", "rand", "rcgen", "ref-cast", @@ -5124,6 +5207,7 @@ dependencies = [ "term", "thiserror", "tokio", + "url", "usdt", "uuid", ] @@ -5141,11 +5225,29 @@ dependencies = [ "serde_json", ] +[[package]] +name = "nexus-external-api" +version = "0.1.0" +dependencies = [ + "anyhow", + "dropshot 0.10.2-dev", + "http 0.2.12", + "hyper 0.14.30", + "ipnetwork", + "nexus-types", + "omicron-common", + "omicron-workspace-hack", + "openapi-manager-types", + "openapiv3", + "oximeter-types", + "oxql-types", +] + [[package]] name = "nexus-internal-api" version = "0.1.0" dependencies = [ - "dropshot", + "dropshot 0.10.2-dev", "nexus-types", "omicron-common", "omicron-uuid-kinds", @@ -5402,7 +5504,7 @@ dependencies = [ "crucible-agent-client", "dns-server", "dns-service-client", - "dropshot", + "dropshot 0.10.2-dev", "futures", "gateway-messages", "gateway-test-utils", @@ -5412,6 +5514,7 @@ dependencies = [ "hyper 0.14.30", "illumos-utils", "internal-dns", + "nexus-client", "nexus-config", "nexus-db-queries", "nexus-sled-agent-shared", @@ -5460,7 +5563,7 @@ dependencies = [ "derive-where", "derive_more", "dns-service-client", - "dropshot", + "dropshot 0.10.2-dev", "futures", "gateway-client", "http 0.2.12", @@ -5786,7 +5889,8 @@ dependencies = [ "chrono", "clap", "clickhouse-admin-api", - "dropshot", + "clickhouse-admin-types", + "dropshot 0.10.2-dev", "expectorate", "http 0.2.12", "illumos-utils", @@ -5823,7 +5927,7 @@ dependencies = [ "cockroach-admin-api", "cockroach-admin-types", "csv", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "http 0.2.12", "illumos-utils", @@ -5865,7 +5969,7 @@ dependencies = [ "camino", "camino-tempfile", "chrono", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "futures", "hex", @@ -5933,7 +6037,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "futures", "libc", @@ -5973,7 +6077,7 @@ dependencies = [ "camino", "chrono", "clap", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "futures", "gateway-api", @@ -6011,6 +6115,38 @@ dependencies = [ "uuid", ] +[[package]] +name = "omicron-live-tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_matches", + "dropshot 0.10.2-dev", + "futures", + "internal-dns", + "live-tests-macros", + "nexus-client", + "nexus-config", + "nexus-db-model", + "nexus-db-queries", + "nexus-reconfigurator-planning", + "nexus-reconfigurator-preparation", + "nexus-sled-agent-shared", + "nexus-types", + "omicron-common", + "omicron-rpaths", + "omicron-test-utils", + "omicron-workspace-hack", + "pq-sys", + "reqwest", + "serde", + "slog", + "slog-error-chain", + "textwrap", + "tokio", + "uuid", +] + [[package]] name = "omicron-nexus" version = "0.1.0" @@ -6037,7 +6173,7 @@ dependencies = [ "dns-server", "dns-service-client", "dpd-client", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "fatfs", "futures", @@ -6064,6 +6200,7 @@ dependencies = [ "nexus-db-model", "nexus-db-queries", "nexus-defaults", + "nexus-external-api", "nexus-internal-api", "nexus-inventory", "nexus-metrics-producer-gc", @@ -6161,7 +6298,7 @@ dependencies = [ "crucible-agent-client", "csv", "diesel", - "dropshot", + "dropshot 0.10.2-dev", "dyn-clone", "expectorate", "futures", @@ -6325,7 +6462,7 @@ dependencies = [ "dns-server", "dns-service-client", "dpd-client", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "flate2", "flume", @@ -6410,7 +6547,7 @@ dependencies = [ "atomicwrites", "camino", "camino-tempfile", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "filetime", "gethostname", @@ -6511,7 +6648,7 @@ dependencies = [ "log", "managed", "memchr", - "mio 0.8.11", + "mio 1.0.2", "nix 0.28.0", "nom", "num-bigint-dig", @@ -6646,11 +6783,12 @@ dependencies = [ "clickhouse-admin-api", "cockroach-admin-api", "dns-server-api", - "dropshot", + "dropshot 0.10.2-dev", "fs-err", "gateway-api", "indent_write", "installinator-api", + "nexus-external-api", "nexus-internal-api", "omicron-workspace-hack", "openapi-lint", @@ -6772,12 +6910,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "oso" version = "0.27.3" @@ -6870,7 +7002,7 @@ name = "oximeter-api" version = "0.1.0" dependencies = [ "chrono", - "dropshot", + "dropshot 0.10.2-dev", "omicron-common", "omicron-workspace-hack", "schemars", @@ -6901,7 +7033,7 @@ dependencies = [ "camino", "chrono", "clap", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "futures", "hyper 0.14.30", @@ -6949,7 +7081,7 @@ dependencies = [ "clap", "clickward", "crossterm 0.28.1", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "futures", "highway", @@ -6991,7 +7123,7 @@ version = "0.1.0" dependencies = [ "cfg-if", "chrono", - "dropshot", + "dropshot 0.10.2-dev", "futures", "http 0.2.12", "hyper 0.14.30", @@ -7027,7 +7159,7 @@ dependencies = [ "anyhow", "chrono", "clap", - "dropshot", + "dropshot 0.10.2-dev", "internal-dns", "nexus-client", "omicron-common", @@ -8022,7 +8154,7 @@ dependencies = [ "atty", "base64 0.21.7", "clap", - "dropshot", + "dropshot 0.10.2-dev", "futures", "hyper 0.14.30", "progenitor", @@ -8109,6 +8241,29 @@ dependencies = [ "psl-types", ] +[[package]] +name = "qorb" +version = "0.0.1" +source = "git+https://github.com/oxidecomputer/qorb?branch=master#163a77838a3cfe8f7741d32e443f76d995b89df3" +dependencies = [ + "anyhow", + "async-trait", + "debug-ignore", + "derive-where", + "dropshot 0.10.1", + "futures", + "hickory-resolver", + "rand", + "schemars", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", + "tokio-tungstenite 0.23.1", + "tracing", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -8282,7 +8437,7 @@ dependencies = [ "camino-tempfile", "clap", "dns-service-client", - "dropshot", + "dropshot 0.10.2-dev", "expectorate", "humantime", "indexmap 2.4.0", @@ -8674,9 +8829,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.44.1" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6500eedfaf8cd81597899d896908a4b9cd5cb566db875e843c04ccf92add2c16" +checksum = "0a229f2a03daea3f62cee897b40329ce548600cca615906d98d58b8db3029b19" dependencies = [ "aes", "aes-gcm", @@ -8687,6 +8842,7 @@ dependencies = [ "chacha20", "ctr", "curve25519-dalek", + "des", "digest", "elliptic-curve", "flate2", @@ -8726,9 +8882,9 @@ dependencies = [ [[package]] name = "russh-keys" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb8c0bfe024d4edd242f65a2ac6c8bf38a892930050b9eb90909d8fc2c413c8d" +checksum = "89757474f7c9ee30121d8cc7fe293a954ba10b204a82ccf5850a5352a532ebc7" dependencies = [ "aes", "async-trait", @@ -8740,12 +8896,12 @@ dependencies = [ "data-encoding", "der", "digest", - "dirs", "ecdsa", "ed25519-dalek", "elliptic-curve", "futures", "hmac", + "home", "inout", "log", "md5", @@ -9538,7 +9694,7 @@ name = "sled-agent-api" version = "0.1.0" dependencies = [ "camino", - "dropshot", + "dropshot 0.10.2-dev", "nexus-sled-agent-shared", "omicron-common", "omicron-uuid-kinds", @@ -9908,7 +10064,7 @@ dependencies = [ "anyhow", "async-trait", "clap", - "dropshot", + "dropshot 0.10.2-dev", "futures", "gateway-messages", "gateway-types", @@ -10632,28 +10788,27 @@ dependencies = [ [[package]] name = "tokio" -version = "1.38.1" +version = "1.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" +checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" dependencies = [ "backtrace", "bytes", "libc", - "mio 0.8.11", - "num_cpus", + "mio 1.0.2", "parking_lot 0.12.2", "pin-project-lite", "signal-hook-registry", "socket2 0.5.7", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", @@ -10726,6 +10881,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -10752,6 +10908,18 @@ dependencies = [ "tungstenite 0.21.0", ] +[[package]] +name = "tokio-tungstenite" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.23.0", +] + [[package]] name = "tokio-util" version = "0.7.11" @@ -11103,6 +11271,24 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.1.0", + "httparse", + "log", + "rand", + "sha1", + "thiserror", + "utf-8", +] + [[package]] name = "twox-hash" version = "1.6.3" @@ -11304,7 +11490,7 @@ dependencies = [ "clap", "debug-ignore", "display-error-chain", - "dropshot", + "dropshot 0.10.2-dev", "futures", "hex", "hubtools", @@ -11773,7 +11959,7 @@ version = "0.1.0" dependencies = [ "anyhow", "dpd-client", - "dropshot", + "dropshot 0.10.2-dev", "gateway-client", "maplit", "omicron-common", @@ -11829,7 +12015,7 @@ dependencies = [ "debug-ignore", "display-error-chain", "dpd-client", - "dropshot", + "dropshot 0.10.2-dev", "either", "expectorate", "flate2", @@ -11896,7 +12082,7 @@ name = "wicketd-api" version = "0.1.0" dependencies = [ "bootstrap-agent-client", - "dropshot", + "dropshot 0.10.2-dev", "gateway-client", "omicron-common", "omicron-passwords", @@ -12201,6 +12387,7 @@ version = "0.1.0" dependencies = [ "anyhow", "camino", + "camino-tempfile", "cargo_metadata", "cargo_toml", "clap", @@ -12209,6 +12396,7 @@ dependencies = [ "serde", "swrite", "tabled", + "textwrap", "toml 0.8.19", "usdt", ] @@ -12352,7 +12540,7 @@ dependencies = [ "anyhow", "camino", "clap", - "dropshot", + "dropshot 0.10.2-dev", "illumos-utils", "omicron-common", "omicron-sled-agent", diff --git a/Cargo.toml b/Cargo.toml index 9e1d235f12..07edb8b4af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,8 @@ members = [ "internal-dns", "ipcc", "key-manager", + "live-tests", + "live-tests/macros", "nexus", "nexus-config", "nexus-sled-agent-shared", @@ -62,6 +64,7 @@ members = [ "nexus/db-model", "nexus/db-queries", "nexus/defaults", + "nexus/external-api", "nexus/internal-api", "nexus/inventory", "nexus/macros-common", @@ -120,6 +123,7 @@ default-members = [ "certificates", "clickhouse-admin", "clickhouse-admin/api", + "clickhouse-admin/types", "clients/bootstrap-agent-client", "clients/cockroach-admin-client", "clients/ddm-admin-client", @@ -155,8 +159,7 @@ default-members = [ # See omicron#4392. "dns-server", "dns-server-api", - # Do not include end-to-end-tests in the list of default members, as its - # tests only work on a deployed control plane. + "end-to-end-tests", "gateway", "gateway-api", "gateway-cli", @@ -170,6 +173,8 @@ default-members = [ "internal-dns", "ipcc", "key-manager", + "live-tests", + "live-tests/macros", "nexus", "nexus-config", "nexus-sled-agent-shared", @@ -180,6 +185,7 @@ default-members = [ "nexus/db-model", "nexus/db-queries", "nexus/defaults", + "nexus/external-api", "nexus/internal-api", "nexus/inventory", "nexus/macros-common", @@ -282,13 +288,12 @@ api_identity = { path = "api_identity" } approx = "0.5.1" assert_matches = "1.5.0" assert_cmd = "2.0.16" -async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "ed7ab5ef0513ba303d33efd41d3e9e381169d59b" } +async-bb8-diesel = "0.2" async-trait = "0.1.81" atomicwrites = "0.4.3" authz-macros = { path = "nexus/authz-macros" } backoff = { version = "0.4.0", features = [ "tokio" ] } base64 = "0.22.1" -bb8 = "0.8.5" bcs = "0.1.6" bincode = "1.3.3" bootstore = { path = "bootstore" } @@ -307,6 +312,7 @@ chrono = { version = "0.4", features = [ "serde" ] } ciborium = "0.2.2" clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] } clickhouse-admin-api = { path = "clickhouse-admin/api" } +clickhouse-admin-types = { path = "clickhouse-admin/types" } clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "ceec762e6a87d2a22bf56792a3025e145caa095e" } cockroach-admin-api = { path = "cockroach-admin/api" } cockroach-admin-client = { path = "clients/cockroach-admin-client" } @@ -324,14 +330,14 @@ crucible-common = { git = "https://github.com/oxidecomputer/crucible", rev = "e5 csv = "1.3.0" curve25519-dalek = "4" datatest-stable = "0.2.9" -display-error-chain = "0.2.0" +display-error-chain = "0.2.1" omicron-ddm-admin-client = { path = "clients/ddm-admin-client" } db-macros = { path = "nexus/db-macros" } debug-ignore = "1.0.5" derive_more = "0.99.18" derive-where = "1.2.7" # Having the i-implement-... feature here makes diesel go away from the workspace-hack -diesel = { version = "2.2.2", features = ["i-implement-a-third-party-backend-and-opt-into-breaking-changes", "postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } +diesel = { version = "2.2.3", features = ["i-implement-a-third-party-backend-and-opt-into-breaking-changes", "postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } diesel-dtrace = { git = "https://github.com/oxidecomputer/diesel-dtrace", branch = "main" } dns-server = { path = "dns-server" } dns-server-api = { path = "dns-server-api" } @@ -401,10 +407,12 @@ ipnetwork = { version = "0.20", features = ["schemars"] } ispf = { git = "https://github.com/oxidecomputer/ispf" } key-manager = { path = "key-manager" } kstat-rs = "0.2.4" -libc = "0.2.156" +libc = "0.2.158" +libipcc = { git = "https://github.com/oxidecomputer/libipcc", rev = "fdffa212373a8f92473ea5f411088912bf458d5f" } libfalcon = { git = "https://github.com/oxidecomputer/falcon", rev = "e69694a1f7cc9fe31fab27f321017280531fb5f7" } libnvme = { git = "https://github.com/oxidecomputer/libnvme", rev = "dd5bb221d327a1bc9287961718c3c10d6bd37da0" } linear-map = "1.2.0" +live-tests-macros = { path = "live-tests/macros" } macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.13" @@ -419,6 +427,7 @@ nexus-db-fixed-data = { path = "nexus/db-fixed-data" } nexus-db-model = { path = "nexus/db-model" } nexus-db-queries = { path = "nexus/db-queries" } nexus-defaults = { path = "nexus/defaults" } +nexus-external-api = { path = "nexus/external-api" } nexus-inventory = { path = "nexus/inventory" } nexus-internal-api = { path = "nexus/internal-api" } nexus-macros-common = { path = "nexus/macros-common" } @@ -498,6 +507,7 @@ bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "24a74d0c propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "24a74d0c76b6a63961ecef76acb1516b6e66c5c9" } propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "24a74d0c76b6a63961ecef76acb1516b6e66c5c9" } proptest = "1.5.0" +qorb = { git = "https://github.com/oxidecomputer/qorb", branch = "master" } quote = "1.0" rand = "0.8.5" rand_core = "0.6.4" @@ -581,7 +591,7 @@ textwrap = "0.16.1" test-strategy = "0.3.1" thiserror = "1.0" tofino = { git = "https://github.com/oxidecomputer/tofino", branch = "main" } -tokio = "1.38.1" +tokio = "1.39.3" tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] } tokio-stream = "0.1.15" tokio-tungstenite = "0.20" diff --git a/README.adoc b/README.adoc index d48a5c9736..80753e030f 100644 --- a/README.adoc +++ b/README.adoc @@ -62,6 +62,8 @@ Nextest https://github.com/nextest-rs/nextest/issues/16[does not support doctest Similarly, you can run tests inside a https://github.com/oxidecomputer/falcon[Falcon] based VM. This is described in the `test-utils` https://github.com/oxidecomputer/omicron/tree/main/test-utils[README]. +There's also a xref:./live-tests/README.adoc[`live-tests`] test suite that can be run by hand in a _deployed_ Omicron system. + === rustfmt and clippy You can **format the code** using `cargo fmt`. Make sure to run this before pushing changes. The CI checks that the code is correctly formatted. @@ -206,12 +208,14 @@ We also use these OpenAPI documents as the source for the clients we generate using https://github.com/oxidecomputer/progenitor[Progenitor]. Clients are automatically updated when the coresponding OpenAPI document is modified. -There are currently two kinds of services based on how their corresponding documents are generated: *managed* and *unmanaged*. Eventually, all services within Omicron will transition to being managed. +OpenAPI documents are tracked by the `cargo xtask openapi` command. -* A *managed* service is tracked by the `cargo xtask openapi` command, using Dropshot's relatively new API trait functionality. -* An *unmanaged* service is defined the traditional way, by gluing together a set of implementation functions, and is tracked by an independent test. +* To regenerate all OpenAPI documents, run `cargo xtask openapi generate`. +* To check whether all OpenAPI documents are up-to-date, run `cargo xtask + openapi check`. -To check whether your document is managed, run `cargo xtask openapi list`: it will list out all managed OpenAPI documents. If your document is not on the list, it is unmanaged. +For more information, see the documentation in +link:./dev-tools/openapi-manager[`dev-tools/openapi-manager`]. Note that Omicron contains a nominally circular dependency: @@ -223,33 +227,6 @@ Note that Omicron contains a nominally circular dependency: We effectively "break" this circular dependency by virtue of the OpenAPI documents being checked in. -==== Updating or Creating New Managed Services - -See the documentation in link:./dev-tools/openapi-manager[`dev-tools/openapi-manager`] for more information. - -==== Updating Unmanaged Services - -In general, changes to unmanaged service APs **require the following set of build steps**: - -. Make changes to the service API. -. Update the OpenAPI document by running the relevant test with overwrite set: - `EXPECTORATE=overwrite cargo nextest run -p -- test_nexus_openapi_internal` - (changing the package name and test name as necessary). It's important to do - this _before_ the next step. -. This will cause the generated client to be updated which may break the build - for dependent consumers. -. Modify any dependent services to fix calls to the generated client. - -Note that if you make changes to both Nexus and Sled Agent simultaneously, you -may end up in a spot where neither can build and therefore neither OpenAPI -document can be generated. In this case, revert or comment out changes in one -so that the OpenAPI document can be generated. - -This is a particular problem if you find yourself resolving merge conflicts in the generated files. You have basically two options for this: - -* Resolve the merge conflicts by hand. This is usually not too bad in practice. -* Take the upstream copy of the file, back out your client side changes (`git stash` and its `-p` option can be helpful for this), follow the steps above to regenerate the file using the automated test, and finally re-apply your changes to the client side. This is essentially getting yourself back to step 1 above and then following the procedure above. - === Resolving merge conflicts in Cargo.lock When pulling in new changes from upstream "main", you may find conflicts in Cargo.lock. The easiest way to deal with these is usually to take the upstream changes as-is, then trigger any Cargo operation that updates the lockfile. `cargo metadata` is a quick one. Here's an example: diff --git a/clickhouse-admin/Cargo.toml b/clickhouse-admin/Cargo.toml index 033836dfe0..270f779d7e 100644 --- a/clickhouse-admin/Cargo.toml +++ b/clickhouse-admin/Cargo.toml @@ -10,6 +10,7 @@ camino.workspace = true chrono.workspace = true clap.workspace = true clickhouse-admin-api.workspace = true +clickhouse-admin-types.workspace = true dropshot.workspace = true http.workspace = true illumos-utils.workspace = true diff --git a/clickhouse-admin/types/Cargo.toml b/clickhouse-admin/types/Cargo.toml new file mode 100644 index 0000000000..a90a7c2e39 --- /dev/null +++ b/clickhouse-admin/types/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "clickhouse-admin-types" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +camino.workspace = true +camino-tempfile.workspace = true +derive_more.workspace = true +omicron-common.workspace = true +omicron-workspace-hack.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true +expectorate.workspace = true diff --git a/clickhouse-admin/types/src/config.rs b/clickhouse-admin/types/src/config.rs new file mode 100644 index 0000000000..3337d733a9 --- /dev/null +++ b/clickhouse-admin/types/src/config.rs @@ -0,0 +1,515 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::{KeeperId, ServerId, OXIMETER_CLUSTER}; +use camino::Utf8PathBuf; +use omicron_common::address::{ + CLICKHOUSE_HTTP_PORT, CLICKHOUSE_INTERSERVER_PORT, + CLICKHOUSE_KEEPER_RAFT_PORT, CLICKHOUSE_KEEPER_TCP_PORT, + CLICKHOUSE_TCP_PORT, +}; +use schemars::{ + gen::SchemaGenerator, + schema::{Schema, SchemaObject}, + JsonSchema, +}; +use serde::{Deserialize, Serialize}; +use std::{fmt::Display, net::Ipv6Addr}; + +// Used for schemars to be able to be used with camino: +// See https://github.com/camino-rs/camino/issues/91#issuecomment-2027908513 +fn path_schema(gen: &mut SchemaGenerator) -> Schema { + let mut schema: SchemaObject = ::json_schema(gen).into(); + schema.format = Some("Utf8PathBuf".to_owned()); + schema.into() +} + +/// Configuration for a ClickHouse replica server +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct ReplicaConfig { + pub logger: LogConfig, + pub macros: Macros, + pub listen_host: Ipv6Addr, + pub http_port: u16, + pub tcp_port: u16, + pub interserver_http_port: u16, + pub remote_servers: RemoteServers, + pub keepers: KeeperConfigsForReplica, + #[schemars(schema_with = "path_schema")] + pub data_path: Utf8PathBuf, +} + +impl ReplicaConfig { + /// A new ClickHouse replica server configuration with default ports and directories + pub fn new( + logger: LogConfig, + macros: Macros, + listen_host: Ipv6Addr, + remote_servers: Vec, + keepers: Vec, + path: Utf8PathBuf, + ) -> Self { + let data_path = path.join("data"); + let remote_servers = RemoteServers::new(remote_servers); + let keepers = KeeperConfigsForReplica::new(keepers); + + Self { + logger, + macros, + listen_host, + http_port: CLICKHOUSE_HTTP_PORT, + tcp_port: CLICKHOUSE_TCP_PORT, + interserver_http_port: CLICKHOUSE_INTERSERVER_PORT, + remote_servers, + keepers, + data_path, + } + } + + pub fn to_xml(&self) -> String { + let ReplicaConfig { + logger, + macros, + listen_host, + http_port, + tcp_port, + interserver_http_port, + remote_servers, + keepers, + data_path, + } = self; + let logger = logger.to_xml(); + let cluster = macros.cluster.clone(); + let id = macros.replica; + let macros = macros.to_xml(); + let keepers = keepers.to_xml(); + let remote_servers = remote_servers.to_xml(); + let user_files_path = data_path.clone().join("user_files"); + let format_schema_path = data_path.clone().join("format_schemas"); + format!( + " + +{logger} + {data_path} + + + + random + + + + + + + + + ::/0 + + default + default + + + + + + + 3600 + 0 + 0 + 0 + 0 + 0 + + + + + {user_files_path} + default + {format_schema_path} + {cluster}_{id} + {listen_host} + {http_port} + {tcp_port} + {interserver_http_port} + {listen_host} + + + + + 604800 + + + 60 + + + 1000 + +{macros} +{remote_servers} +{keepers} + + +" + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct Macros { + pub shard: u64, + pub replica: ServerId, + pub cluster: String, +} + +impl Macros { + /// A new macros configuration block with default cluster + pub fn new(replica: ServerId) -> Self { + Self { shard: 1, replica, cluster: OXIMETER_CLUSTER.to_string() } + } + + pub fn to_xml(&self) -> String { + let Macros { shard, replica, cluster } = self; + format!( + " + + {shard} + {replica} + {cluster} + " + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct RemoteServers { + pub cluster: String, + pub secret: String, + pub replicas: Vec, +} + +impl RemoteServers { + /// A new remote_servers configuration block with default cluster + pub fn new(replicas: Vec) -> Self { + Self { + cluster: OXIMETER_CLUSTER.to_string(), + // TODO(https://github.com/oxidecomputer/omicron/issues/3823): secret handling TBD + secret: "some-unique-value".to_string(), + replicas, + } + } + + pub fn to_xml(&self) -> String { + let RemoteServers { cluster, secret, replicas } = self; + + let mut s = format!( + " + + <{cluster}> + + {secret} + + true" + ); + + for r in replicas { + let ServerNodeConfig { host, port } = r; + s.push_str(&format!( + " + + {host} + {port} + " + )); + } + + s.push_str(&format!( + " + + + + " + )); + + s + } +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct KeeperConfigsForReplica { + pub nodes: Vec, +} + +impl KeeperConfigsForReplica { + pub fn new(nodes: Vec) -> Self { + Self { nodes } + } + + pub fn to_xml(&self) -> String { + let mut s = String::from(" "); + for node in &self.nodes { + let KeeperNodeConfig { host, port } = node; + + // ClickHouse servers have a small quirk, where when setting the + // keeper hosts as IPv6 addresses in the replica configuration file, + // they must be wrapped in square brackets. + // Otherwise, when running any query, a "Service not found" error + // appears. + // https://github.com/ClickHouse/ClickHouse/blob/a011990fd75628c63c7995c4f15475f1d4125d10/src/Coordination/KeeperStateManager.cpp#L149 + let parsed_host = match host.parse::() { + Ok(_) => format!("[{host}]"), + Err(_) => host.to_string(), + }; + + s.push_str(&format!( + " + + {parsed_host} + {port} + ", + )); + } + s.push_str("\n "); + s + } +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct KeeperNodeConfig { + pub host: String, + pub port: u16, +} + +impl KeeperNodeConfig { + /// A new ClickHouse keeper node configuration with default port + pub fn new(host: String) -> Self { + let port = CLICKHOUSE_KEEPER_TCP_PORT; + Self { host, port } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct ServerNodeConfig { + pub host: String, + pub port: u16, +} + +impl ServerNodeConfig { + /// A new ClickHouse replica node configuration with default port + pub fn new(host: String) -> Self { + let port = CLICKHOUSE_TCP_PORT; + Self { host, port } + } +} + +pub enum NodeType { + Server, + Keeper, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct LogConfig { + pub level: LogLevel, + #[schemars(schema_with = "path_schema")] + pub log: Utf8PathBuf, + #[schemars(schema_with = "path_schema")] + pub errorlog: Utf8PathBuf, + pub size: u16, + pub count: usize, +} + +impl LogConfig { + /// A new logger configuration with default directories + pub fn new(path: Utf8PathBuf, node_type: NodeType) -> Self { + let prefix = match node_type { + NodeType::Server => "clickhouse", + NodeType::Keeper => "clickhouse-keeper", + }; + + let logs: Utf8PathBuf = path.join("log"); + let log = logs.join(format!("{prefix}.log")); + let errorlog = logs.join(format!("{prefix}.err.log")); + + Self { level: LogLevel::default(), log, errorlog, size: 100, count: 1 } + } + + pub fn to_xml(&self) -> String { + let LogConfig { level, log, errorlog, size, count } = &self; + format!( + " + + {level} + {log} + {errorlog} + {size}M + {count} + +" + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct KeeperCoordinationSettings { + pub operation_timeout_ms: u32, + pub session_timeout_ms: u32, + pub raft_logs_level: LogLevel, +} + +impl KeeperCoordinationSettings { + pub fn default() -> Self { + Self { + operation_timeout_ms: 10000, + session_timeout_ms: 30000, + raft_logs_level: LogLevel::Trace, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct RaftServers { + pub servers: Vec, +} + +impl RaftServers { + pub fn new(servers: Vec) -> Self { + Self { servers } + } + pub fn to_xml(&self) -> String { + let mut s = String::new(); + for server in &self.servers { + let RaftServerConfig { id, hostname, port } = server; + s.push_str(&format!( + " + + {id} + {hostname} + {port} + + " + )); + } + + s + } +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct RaftServerConfig { + pub id: KeeperId, + pub hostname: String, + pub port: u16, +} + +impl RaftServerConfig { + pub fn new(id: KeeperId, hostname: String) -> Self { + Self { id, hostname, port: CLICKHOUSE_KEEPER_RAFT_PORT } + } +} + +/// Configuration for a ClickHouse keeper +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct KeeperConfig { + pub logger: LogConfig, + pub listen_host: Ipv6Addr, + pub tcp_port: u16, + pub server_id: KeeperId, + #[schemars(schema_with = "path_schema")] + pub log_storage_path: Utf8PathBuf, + #[schemars(schema_with = "path_schema")] + pub snapshot_storage_path: Utf8PathBuf, + pub coordination_settings: KeeperCoordinationSettings, + pub raft_config: RaftServers, +} + +impl KeeperConfig { + /// A new ClickHouse keeper node configuration with default ports and directories + pub fn new( + logger: LogConfig, + listen_host: Ipv6Addr, + server_id: KeeperId, + datastore_path: Utf8PathBuf, + raft_config: RaftServers, + ) -> Self { + let coordination_path = datastore_path.join("coordination"); + let log_storage_path = coordination_path.join("log"); + let snapshot_storage_path = coordination_path.join("snapshots"); + let coordination_settings = KeeperCoordinationSettings::default(); + Self { + logger, + listen_host, + tcp_port: CLICKHOUSE_KEEPER_TCP_PORT, + server_id, + log_storage_path, + snapshot_storage_path, + coordination_settings, + raft_config, + } + } + + pub fn to_xml(&self) -> String { + let KeeperConfig { + logger, + listen_host, + tcp_port, + server_id, + log_storage_path, + snapshot_storage_path, + coordination_settings, + raft_config, + } = self; + let logger = logger.to_xml(); + let KeeperCoordinationSettings { + operation_timeout_ms, + session_timeout_ms, + raft_logs_level, + } = coordination_settings; + let raft_servers = raft_config.to_xml(); + format!( + " + +{logger} + {listen_host} + + false + {tcp_port} + {server_id} + {log_storage_path} + {snapshot_storage_path} + + {operation_timeout_ms} + {session_timeout_ms} + {raft_logs_level} + + +{raft_servers} + + + + +" + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub enum LogLevel { + Trace, + Debug, +} + +impl LogLevel { + fn default() -> Self { + LogLevel::Trace + } +} + +impl Display for LogLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + LogLevel::Trace => "trace", + LogLevel::Debug => "debug", + }; + write!(f, "{s}") + } +} diff --git a/clickhouse-admin/types/src/lib.rs b/clickhouse-admin/types/src/lib.rs new file mode 100644 index 0000000000..c9cc076de5 --- /dev/null +++ b/clickhouse-admin/types/src/lib.rs @@ -0,0 +1,242 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Result; +use camino::Utf8PathBuf; +use camino_tempfile::NamedUtf8TempFile; +use derive_more::{Add, AddAssign, Display, From}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::fs::rename; +use std::io::Write; +use std::net::Ipv6Addr; + +pub mod config; +use config::*; + +pub const OXIMETER_CLUSTER: &str = "oximeter_cluster"; + +/// A unique ID for a ClickHouse Keeper +#[derive( + Debug, + Clone, + Copy, + Eq, + PartialEq, + Ord, + PartialOrd, + From, + Add, + AddAssign, + Display, + JsonSchema, + Serialize, + Deserialize, +)] +pub struct KeeperId(pub u64); + +/// A unique ID for a Clickhouse Server +#[derive( + Debug, + Clone, + Copy, + Eq, + PartialEq, + Ord, + PartialOrd, + From, + Add, + AddAssign, + Display, + JsonSchema, + Serialize, + Deserialize, +)] +pub struct ServerId(pub u64); + +#[derive(Debug, Clone)] +pub struct ClickhouseServerConfig { + pub config_dir: Utf8PathBuf, + pub id: ServerId, + pub datastore_path: Utf8PathBuf, + pub listen_addr: Ipv6Addr, + pub keepers: Vec, + pub servers: Vec, +} + +impl ClickhouseServerConfig { + pub fn new( + config_dir: Utf8PathBuf, + id: ServerId, + datastore_path: Utf8PathBuf, + listen_addr: Ipv6Addr, + keepers: Vec, + servers: Vec, + ) -> Self { + Self { config_dir, id, datastore_path, listen_addr, keepers, servers } + } + + /// Generate a configuration file for a replica server node + pub fn generate_xml_file(&self) -> Result<()> { + let logger = + LogConfig::new(self.datastore_path.clone(), NodeType::Server); + let macros = Macros::new(self.id); + + let config = ReplicaConfig::new( + logger, + macros, + self.listen_addr, + self.servers.clone(), + self.keepers.clone(), + self.datastore_path.clone(), + ); + + // Writing to a temporary file and then renaming it will ensure we + // don't end up with a partially written file after a crash + let mut f = NamedUtf8TempFile::new()?; + f.write_all(config.to_xml().as_bytes())?; + f.flush()?; + rename(f.path(), self.config_dir.join("replica-server-config.xml"))?; + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub struct ClickhouseKeeperConfig { + pub config_dir: Utf8PathBuf, + pub id: KeeperId, + pub raft_servers: Vec, + pub datastore_path: Utf8PathBuf, + pub listen_addr: Ipv6Addr, +} + +impl ClickhouseKeeperConfig { + pub fn new( + config_dir: Utf8PathBuf, + id: KeeperId, + raft_servers: Vec, + datastore_path: Utf8PathBuf, + listen_addr: Ipv6Addr, + ) -> Self { + ClickhouseKeeperConfig { + config_dir, + id, + raft_servers, + datastore_path, + listen_addr, + } + } + + /// Generate a configuration file for a keeper node + pub fn generate_xml_file(&self) -> Result<()> { + let logger = + LogConfig::new(self.datastore_path.clone(), NodeType::Keeper); + let raft_config = RaftServers::new(self.raft_servers.clone()); + let config = KeeperConfig::new( + logger, + self.listen_addr, + self.id, + self.datastore_path.clone(), + raft_config, + ); + + // Writing to a temporary file and then renaming it will ensure we + // don't end up with a partially written file after a crash + let mut f = NamedUtf8TempFile::new()?; + f.write_all(config.to_xml().as_bytes())?; + f.flush()?; + rename(f.path(), self.config_dir.join("keeper-config.xml"))?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{net::Ipv6Addr, str::FromStr}; + + use camino::Utf8PathBuf; + use camino_tempfile::Builder; + + use crate::{ + ClickhouseKeeperConfig, ClickhouseServerConfig, KeeperId, + KeeperNodeConfig, RaftServerConfig, ServerId, ServerNodeConfig, + }; + + #[test] + fn test_generate_keeper_config() { + let config_dir = Builder::new() + .tempdir_in( + Utf8PathBuf::try_from(std::env::temp_dir()).unwrap()) + .expect("Could not create directory for ClickHouse configuration generation test" + ); + + let keepers = vec![ + RaftServerConfig::new(KeeperId(1), "ff::01".to_string()), + RaftServerConfig::new(KeeperId(2), "ff::02".to_string()), + RaftServerConfig::new(KeeperId(3), "ff::03".to_string()), + ]; + + let config = ClickhouseKeeperConfig::new( + Utf8PathBuf::from(config_dir.path()), + KeeperId(1), + keepers, + Utf8PathBuf::from_str("./").unwrap(), + Ipv6Addr::from_str("ff::08").unwrap(), + ); + + config.generate_xml_file().unwrap(); + + let expected_file = Utf8PathBuf::from_str("./testutils") + .unwrap() + .join("keeper-config.xml"); + let generated_file = + Utf8PathBuf::from(config_dir.path()).join("keeper-config.xml"); + let generated_content = std::fs::read_to_string(generated_file) + .expect("Failed to read from generated ClickHouse keeper file"); + + expectorate::assert_contents(expected_file, &generated_content); + } + + #[test] + fn test_generate_replica_config() { + let config_dir = Builder::new() + .tempdir_in( + Utf8PathBuf::try_from(std::env::temp_dir()).unwrap()) + .expect("Could not create directory for ClickHouse configuration generation test" + ); + + let keepers = vec![ + KeeperNodeConfig::new("ff::01".to_string()), + KeeperNodeConfig::new("127.0.0.1".to_string()), + KeeperNodeConfig::new("we.dont.want.brackets.com".to_string()), + ]; + + let servers = vec![ + ServerNodeConfig::new("ff::08".to_string()), + ServerNodeConfig::new("ff::09".to_string()), + ]; + + let config = ClickhouseServerConfig::new( + Utf8PathBuf::from(config_dir.path()), + ServerId(1), + Utf8PathBuf::from_str("./").unwrap(), + Ipv6Addr::from_str("ff::08").unwrap(), + keepers, + servers, + ); + + config.generate_xml_file().unwrap(); + + let expected_file = Utf8PathBuf::from_str("./testutils") + .unwrap() + .join("replica-server-config.xml"); + let generated_file = Utf8PathBuf::from(config_dir.path()) + .join("replica-server-config.xml"); + let generated_content = std::fs::read_to_string(generated_file).expect( + "Failed to read from generated ClickHouse replica server file", + ); + + expectorate::assert_contents(expected_file, &generated_content); + } +} diff --git a/clickhouse-admin/types/testutils/keeper-config.xml b/clickhouse-admin/types/testutils/keeper-config.xml new file mode 100644 index 0000000000..e05cf9d954 --- /dev/null +++ b/clickhouse-admin/types/testutils/keeper-config.xml @@ -0,0 +1,47 @@ + + + + + trace + ./log/clickhouse-keeper.log + ./log/clickhouse-keeper.err.log + 100M + 1 + + + ff::8 + + false + 9181 + 1 + ./coordination/log + ./coordination/snapshots + + 10000 + 30000 + trace + + + + + 1 + ff::01 + 9234 + + + + 2 + ff::02 + 9234 + + + + 3 + ff::03 + 9234 + + + + + + diff --git a/clickhouse-admin/types/testutils/replica-server-config.xml b/clickhouse-admin/types/testutils/replica-server-config.xml new file mode 100644 index 0000000000..056fd2cc1c --- /dev/null +++ b/clickhouse-admin/types/testutils/replica-server-config.xml @@ -0,0 +1,106 @@ + + + + + trace + ./log/clickhouse.log + ./log/clickhouse.err.log + 100M + 1 + + + ./data + + + + random + + + + + + + + + ::/0 + + default + default + + + + + + + 3600 + 0 + 0 + 0 + 0 + 0 + + + + + ./data/user_files + default + ./data/format_schemas + oximeter_cluster_1 + ff::8 + 8123 + 9000 + 9009 + ff::8 + + + + + 604800 + + + 60 + + + 1000 + + + + 1 + 1 + oximeter_cluster + + + + + + some-unique-value + + true + + ff::08 + 9000 + + + ff::09 + 9000 + + + + + + + + [ff::01] + 9181 + + + 127.0.0.1 + 9181 + + + we.dont.want.brackets.com + 9181 + + + + diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs index a55c5d4013..97f6373e29 100644 --- a/clients/nexus-client/src/lib.rs +++ b/clients/nexus-client/src/lib.rs @@ -131,14 +131,11 @@ impl From } } -impl From - for types::SledInstanceState +impl From + for types::SledVmmState { - fn from( - s: omicron_common::api::internal::nexus::SledInstanceState, - ) -> Self { + fn from(s: omicron_common::api::internal::nexus::SledVmmState) -> Self { Self { - propolis_id: s.propolis_id, vmm_state: s.vmm_state.into(), migration_in: s.migration_in.map(Into::into), migration_out: s.migration_out.map(Into::into), diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index ed96d762dc..be19659c69 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -5,6 +5,7 @@ //! Interface for making API requests to a Sled Agent use async_trait::async_trait; +use omicron_uuid_kinds::PropolisUuid; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -42,6 +43,7 @@ progenitor::generate_api!( replace = { Baseboard = nexus_sled_agent_shared::inventory::Baseboard, ByteCount = omicron_common::api::external::ByteCount, + DatasetKind = omicron_common::api::internal::shared::DatasetKind, DiskIdentity = omicron_common::disk::DiskIdentity, DiskVariant = omicron_common::disk::DiskVariant, Generation = omicron_common::api::external::Generation, @@ -161,12 +163,11 @@ impl From } } -impl From - for omicron_common::api::internal::nexus::SledInstanceState +impl From + for omicron_common::api::internal::nexus::SledVmmState { - fn from(s: types::SledInstanceState) -> Self { + fn from(s: types::SledVmmState) -> Self { Self { - propolis_id: s.propolis_id, vmm_state: s.vmm_state.into(), migration_in: s.migration_in.map(Into::into), migration_out: s.migration_out.map(Into::into), @@ -448,11 +449,11 @@ impl From /// are bonus endpoints, not generated in the real client. #[async_trait] pub trait TestInterfaces { - async fn instance_single_step(&self, id: Uuid); - async fn instance_finish_transition(&self, id: Uuid); - async fn instance_simulate_migration_source( + async fn vmm_single_step(&self, id: PropolisUuid); + async fn vmm_finish_transition(&self, id: PropolisUuid); + async fn vmm_simulate_migration_source( &self, - id: Uuid, + id: PropolisUuid, params: SimulateMigrationSource, ); async fn disk_finish_transition(&self, id: Uuid); @@ -460,10 +461,10 @@ pub trait TestInterfaces { #[async_trait] impl TestInterfaces for Client { - async fn instance_single_step(&self, id: Uuid) { + async fn vmm_single_step(&self, id: PropolisUuid) { let baseurl = self.baseurl(); let client = self.client(); - let url = format!("{}/instances/{}/poke-single-step", baseurl, id); + let url = format!("{}/vmms/{}/poke-single-step", baseurl, id); client .post(url) .send() @@ -471,10 +472,10 @@ impl TestInterfaces for Client { .expect("instance_single_step() failed unexpectedly"); } - async fn instance_finish_transition(&self, id: Uuid) { + async fn vmm_finish_transition(&self, id: PropolisUuid) { let baseurl = self.baseurl(); let client = self.client(); - let url = format!("{}/instances/{}/poke", baseurl, id); + let url = format!("{}/vmms/{}/poke", baseurl, id); client .post(url) .send() @@ -493,14 +494,14 @@ impl TestInterfaces for Client { .expect("disk_finish_transition() failed unexpectedly"); } - async fn instance_simulate_migration_source( + async fn vmm_simulate_migration_source( &self, - id: Uuid, + id: PropolisUuid, params: SimulateMigrationSource, ) { let baseurl = self.baseurl(); let client = self.client(); - let url = format!("{baseurl}/instances/{id}/sim-migration-source"); + let url = format!("{baseurl}/vmms/{id}/sim-migration-source"); client .post(url) .json(¶ms) diff --git a/common/src/address.rs b/common/src/address.rs index c23f5c41ed..49684f0d99 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -8,7 +8,7 @@ //! and Nexus, who need to agree upon addressing schemes. use crate::api::external::{self, Error}; -use crate::policy::{DNS_REDUNDANCY, MAX_DNS_REDUNDANCY}; +use crate::policy::{INTERNAL_DNS_REDUNDANCY, MAX_INTERNAL_DNS_REDUNDANCY}; use ipnetwork::Ipv6Network; use once_cell::sync::Lazy; use oxnet::{Ipv4Net, Ipv6Net}; @@ -33,8 +33,11 @@ pub const SLED_AGENT_PORT: u16 = 12345; pub const COCKROACH_PORT: u16 = 32221; pub const COCKROACH_ADMIN_PORT: u16 = 32222; pub const CRUCIBLE_PORT: u16 = 32345; -pub const CLICKHOUSE_PORT: u16 = 8123; -pub const CLICKHOUSE_KEEPER_PORT: u16 = 9181; +pub const CLICKHOUSE_HTTP_PORT: u16 = 8123; +pub const CLICKHOUSE_INTERSERVER_PORT: u16 = 9009; +pub const CLICKHOUSE_TCP_PORT: u16 = 9000; +pub const CLICKHOUSE_KEEPER_TCP_PORT: u16 = 9181; +pub const CLICKHOUSE_KEEPER_RAFT_PORT: u16 = 9234; pub const CLICKHOUSE_ADMIN_PORT: u16 = 8888; pub const OXIMETER_PORT: u16 = 12223; pub const DENDRITE_PORT: u16 = 12224; @@ -172,7 +175,18 @@ pub const CP_SERVICES_RESERVED_ADDRESSES: u16 = 0xFFFF; pub const SLED_RESERVED_ADDRESSES: u16 = 32; /// Wraps an [`Ipv6Net`] with a compile-time prefix length. -#[derive(Debug, Clone, Copy, JsonSchema, Serialize, Hash, PartialEq, Eq)] +#[derive( + Debug, + Clone, + Copy, + JsonSchema, + Serialize, + Hash, + PartialEq, + Eq, + PartialOrd, + Ord, +)] #[schemars(rename = "Ipv6Subnet")] pub struct Ipv6Subnet { net: Ipv6Net, @@ -226,12 +240,33 @@ impl<'de, const N: u8> Deserialize<'de> for Ipv6Subnet { } /// Represents a subnet which may be used for contacting DNS services. -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[derive( + Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd, Ord, +)] pub struct DnsSubnet { subnet: Ipv6Subnet, } impl DnsSubnet { + pub fn new(subnet: Ipv6Subnet) -> Self { + Self { subnet } + } + + /// Makes a new DNS subnet from the high-order bits of an address. + pub fn from_addr(addr: Ipv6Addr) -> Self { + Self::new(Ipv6Subnet::new(addr)) + } + + /// Returns the DNS subnet. + pub fn subnet(&self) -> Ipv6Subnet { + self.subnet + } + + /// Returns the reserved rack subnet that contains this DNS subnet. + pub fn rack_subnet(&self) -> ReservedRackSubnet { + ReservedRackSubnet::from_subnet(self.subnet) + } + /// Returns the DNS server address within the subnet. /// /// This is the first address within the subnet. @@ -250,7 +285,7 @@ impl DnsSubnet { /// A wrapper around an IPv6 network, indicating it is a "reserved" rack /// subnet which can be used for AZ-wide services. -#[derive(Debug, Clone)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct ReservedRackSubnet(pub Ipv6Subnet); impl ReservedRackSubnet { @@ -259,17 +294,23 @@ impl ReservedRackSubnet { ReservedRackSubnet(Ipv6Subnet::::new(subnet.net().addr())) } + /// Infer the reserved rack subnet from a sled/AZ/DNS subnet. + pub fn from_subnet(subnet: Ipv6Subnet) -> Self { + Self::new(Ipv6Subnet::::new(subnet.net().addr())) + } + + /// Returns the `index`th DNS subnet from this reserved rack subnet. + pub fn get_dns_subnet(&self, index: u8) -> DnsSubnet { + DnsSubnet::new(get_64_subnet(self.0, index)) + } + /// Returns the DNS addresses from this reserved rack subnet. /// - /// These addresses will come from the first [`MAX_DNS_REDUNDANCY`] `/64s` of the - /// [`RACK_PREFIX`] subnet. + /// These addresses will come from the first [`MAX_INTERNAL_DNS_REDUNDANCY`] + /// `/64s` of the [`RACK_PREFIX`] subnet. pub fn get_dns_subnets(&self) -> Vec { - (0..MAX_DNS_REDUNDANCY) - .map(|idx| { - let subnet = - get_64_subnet(self.0, u8::try_from(idx + 1).unwrap()); - DnsSubnet { subnet } - }) + (0..MAX_INTERNAL_DNS_REDUNDANCY) + .map(|idx| self.get_dns_subnet(u8::try_from(idx + 1).unwrap())) .collect() } } @@ -280,7 +321,7 @@ pub fn get_internal_dns_server_addresses(addr: Ipv6Addr) -> Vec { let az_subnet = Ipv6Subnet::::new(addr); let reserved_rack_subnet = ReservedRackSubnet::new(az_subnet); let dns_subnets = - &reserved_rack_subnet.get_dns_subnets()[0..DNS_REDUNDANCY]; + &reserved_rack_subnet.get_dns_subnets()[0..INTERNAL_DNS_REDUNDANCY]; dns_subnets .iter() .map(|dns_subnet| IpAddr::from(dns_subnet.dns_address())) @@ -661,7 +702,7 @@ mod test { // Observe the first DNS subnet within this reserved rack subnet. let dns_subnets = rack_subnet.get_dns_subnets(); - assert_eq!(MAX_DNS_REDUNDANCY, dns_subnets.len()); + assert_eq!(MAX_INTERNAL_DNS_REDUNDANCY, dns_subnets.len()); // The DNS address and GZ address should be only differing by one. assert_eq!( diff --git a/common/src/api/internal/nexus.rs b/common/src/api/internal/nexus.rs index 4daea6a198..191716d017 100644 --- a/common/src/api/internal/nexus.rs +++ b/common/src/api/internal/nexus.rs @@ -113,13 +113,9 @@ pub struct VmmRuntimeState { pub time_updated: DateTime, } -/// A wrapper type containing a sled's total knowledge of the state of a -/// specific VMM and the instance it incarnates. +/// A wrapper type containing a sled's total knowledge of the state of a VMM. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct SledInstanceState { - /// The ID of the VMM whose state is being reported. - pub propolis_id: PropolisUuid, - +pub struct SledVmmState { /// The most recent state of the sled's VMM process. pub vmm_state: VmmRuntimeState, @@ -142,7 +138,7 @@ impl Migrations<'_> { } } -impl SledInstanceState { +impl SledVmmState { pub fn migrations(&self) -> Migrations<'_> { Migrations { migration_in: self.migration_in.as_ref(), @@ -279,31 +275,15 @@ pub struct UpdateArtifactId { // Adding a new KnownArtifactKind // =============================== // -// Adding a new update artifact kind is a tricky process. To do so: +// To add a new kind of update artifact: // // 1. Add it here. +// 2. Regenerate OpenAPI documents with `cargo xtask openapi generate` -- this +// should work without any compile errors. +// 3. Run `cargo check --all-targets` to resolve compile errors. // -// 2. Add the new kind to /clients/src/lib.rs. -// The mapping from `UpdateArtifactKind::*` to `types::UpdateArtifactKind::*` -// must be left as a `todo!()` for now; `types::UpdateArtifactKind` will not -// be updated with the new variant until step 5 below. -// -// 4. Add the new kind and the mapping to its `update_artifact_kind` to -// /nexus/db-model/src/update_artifact.rs -// -// 5. Regenerate the OpenAPI specs for nexus and sled-agent: -// -// ``` -// EXPECTORATE=overwrite cargo nextest run -p omicron-nexus -p omicron-sled-agent openapi -// ``` -// -// 6. Return to /{nexus-client,sled-agent-client}/lib.rs from step 2 -// and replace the `todo!()`s with the new `types::UpdateArtifactKind::*` -// variant. -// -// See https://github.com/oxidecomputer/omicron/pull/2300 as an example. -// -// NOTE: KnownArtifactKind has to be in snake_case due to openapi-lint requirements. +// NOTE: KnownArtifactKind has to be in snake_case due to openapi-lint +// requirements. /// Kinds of update artifacts, as used by Nexus to determine what updates are available and by /// sled-agent to determine how to apply an update when asked. diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 5945efe16d..4826292863 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -10,13 +10,14 @@ use crate::{ }; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use std::{ collections::{HashMap, HashSet}, fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr}, str::FromStr, }; +use strum::EnumCount; use uuid::Uuid; use super::nexus::HostIdentifier; @@ -837,13 +838,11 @@ pub struct ResolvedVpcRouteSet { } /// Describes the purpose of the dataset. -#[derive( - Debug, Serialize, Deserialize, JsonSchema, Clone, Copy, PartialEq, Eq, -)] -#[serde(rename_all = "snake_case")] +#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, EnumCount)] pub enum DatasetKind { - Crucible, + // Durable datasets for zones Cockroach, + Crucible, /// Used for single-node clickhouse deployments Clickhouse, /// Used for replicated clickhouse deployments @@ -852,24 +851,153 @@ pub enum DatasetKind { ClickhouseServer, ExternalDns, InternalDns, + + // Zone filesystems + ZoneRoot, + Zone { + name: String, + }, + + // Other datasets + Debug, +} + +impl Serialize for DatasetKind { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for DatasetKind { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + s.parse().map_err(de::Error::custom) + } +} + +impl JsonSchema for DatasetKind { + fn schema_name() -> String { + "DatasetKind".to_string() + } + + fn json_schema( + gen: &mut schemars::gen::SchemaGenerator, + ) -> schemars::schema::Schema { + // The schema is a bit more complicated than this -- it's either one of + // the fixed values or a string starting with "zone/" -- but this is + // good enough for now. + let mut schema = ::json_schema(gen).into_object(); + schema.metadata().description = Some( + "The kind of dataset. See the `DatasetKind` enum \ + in omicron-common for possible values." + .to_owned(), + ); + schema.into() + } +} + +impl DatasetKind { + pub fn dataset_should_be_encrypted(&self) -> bool { + match self { + // We encrypt all datasets except Crucible. + // + // Crucible already performs encryption internally, and we + // avoid double-encryption. + DatasetKind::Crucible => false, + _ => true, + } + } + + /// Returns true if this dataset is delegated to a non-global zone. + pub fn zoned(&self) -> bool { + use DatasetKind::*; + match self { + Cockroach | Crucible | Clickhouse | ClickhouseKeeper + | ClickhouseServer | ExternalDns | InternalDns => true, + ZoneRoot | Zone { .. } | Debug => false, + } + } + + /// Returns the zone name, if this is a dataset for a zone filesystem. + /// + /// Otherwise, returns "None". + pub fn zone_name(&self) -> Option<&str> { + if let DatasetKind::Zone { name } = self { + Some(name) + } else { + None + } + } } +// Be cautious updating this implementation: +// +// - It should align with [DatasetKind::FromStr], below +// - The strings here are used here comprise the dataset name, stored durably +// on-disk impl fmt::Display for DatasetKind { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { use DatasetKind::*; let s = match self { Crucible => "crucible", - Cockroach => "cockroach", + Cockroach => "cockroachdb", Clickhouse => "clickhouse", ClickhouseKeeper => "clickhouse_keeper", ClickhouseServer => "clickhouse_server", ExternalDns => "external_dns", InternalDns => "internal_dns", + ZoneRoot => "zone", + Zone { name } => { + write!(f, "zone/{}", name)?; + return Ok(()); + } + Debug => "debug", }; write!(f, "{}", s) } } +#[derive(Debug, thiserror::Error)] +pub enum DatasetKindParseError { + #[error("Dataset unknown: {0}")] + UnknownDataset(String), +} + +impl FromStr for DatasetKind { + type Err = DatasetKindParseError; + + fn from_str(s: &str) -> Result { + use DatasetKind::*; + let kind = match s { + "cockroachdb" => Cockroach, + "crucible" => Crucible, + "clickhouse" => Clickhouse, + "clickhouse_keeper" => ClickhouseKeeper, + "clickhouse_server" => ClickhouseServer, + "external_dns" => ExternalDns, + "internal_dns" => InternalDns, + "zone" => ZoneRoot, + "debug" => Debug, + other => { + if let Some(name) = other.strip_prefix("zone/") { + Zone { name: name.to_string() } + } else { + return Err(DatasetKindParseError::UnknownDataset( + s.to_string(), + )); + } + } + }; + Ok(kind) + } +} + /// Identifiers for a single sled. /// /// This is intended primarily to be used in timeseries, to identify @@ -892,6 +1020,7 @@ pub struct SledIdentifiers { #[cfg(test)] mod tests { + use super::*; use crate::api::internal::shared::AllowedSourceIps; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; use std::net::{Ipv4Addr, Ipv6Addr}; @@ -936,4 +1065,49 @@ mod tests { serde_json::from_str(r#"{"allow":"any"}"#).unwrap(), ); } + + #[test] + fn test_dataset_kind_serialization() { + let kinds = [ + DatasetKind::Cockroach, + DatasetKind::Crucible, + DatasetKind::Clickhouse, + DatasetKind::ClickhouseKeeper, + DatasetKind::ClickhouseServer, + DatasetKind::ExternalDns, + DatasetKind::InternalDns, + DatasetKind::ZoneRoot, + DatasetKind::Zone { name: String::from("myzone") }, + DatasetKind::Debug, + ]; + + assert_eq!(kinds.len(), DatasetKind::COUNT); + + for kind in &kinds { + // To string, from string + let as_str = kind.to_string(); + let from_str = + DatasetKind::from_str(&as_str).unwrap_or_else(|_| { + panic!("Failed to convert {kind} to and from string") + }); + assert_eq!( + *kind, from_str, + "{kind} failed to convert to/from a string" + ); + + // Serialize, deserialize + let ser = serde_json::to_string(&kind) + .unwrap_or_else(|_| panic!("Failed to serialize {kind}")); + let de: DatasetKind = serde_json::from_str(&ser) + .unwrap_or_else(|_| panic!("Failed to deserialize {kind}")); + assert_eq!(*kind, de, "{kind} failed serialization"); + + // Test that serialization is equivalent to stringifying. + assert_eq!( + format!("\"{as_str}\""), + ser, + "{kind} does not match stringification/serialization" + ); + } + } } diff --git a/common/src/disk.rs b/common/src/disk.rs index d8b4c2e0a1..ed0bf8666e 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -4,18 +4,23 @@ //! Disk related types shared among crates -use std::fmt; - use anyhow::bail; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::ZpoolUuid; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fmt; use uuid::Uuid; use crate::{ - api::external::Generation, ledger::Ledgerable, zpool_name::ZpoolKind, + api::external::Generation, + ledger::Ledgerable, + zpool_name::{ZpoolKind, ZpoolName}, }; +pub use crate::api::internal::shared::DatasetKind; + #[derive( Clone, Debug, @@ -72,6 +77,243 @@ impl OmicronPhysicalDisksConfig { } } +#[derive( + Debug, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + Clone, + JsonSchema, + PartialOrd, + Ord, +)] +pub struct DatasetName { + // A unique identifier for the Zpool on which the dataset is stored. + pool_name: ZpoolName, + // A name for the dataset within the Zpool. + kind: DatasetKind, +} + +impl DatasetName { + pub fn new(pool_name: ZpoolName, kind: DatasetKind) -> Self { + Self { pool_name, kind } + } + + pub fn pool(&self) -> &ZpoolName { + &self.pool_name + } + + pub fn dataset(&self) -> &DatasetKind { + &self.kind + } + + /// Returns the full name of the dataset, as would be returned from + /// "zfs get" or "zfs list". + /// + /// If this dataset should be encrypted, this automatically adds the + /// "crypt" dataset component. + pub fn full_name(&self) -> String { + // Currently, we encrypt all datasets except Crucible. + // + // Crucible already performs encryption internally, and we + // avoid double-encryption. + if self.kind.dataset_should_be_encrypted() { + self.full_encrypted_name() + } else { + self.full_unencrypted_name() + } + } + + fn full_encrypted_name(&self) -> String { + format!("{}/crypt/{}", self.pool_name, self.kind) + } + + fn full_unencrypted_name(&self) -> String { + format!("{}/{}", self.pool_name, self.kind) + } +} + +#[derive( + Copy, + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, +)] +pub struct GzipLevel(u8); + +// Fastest compression level +const GZIP_LEVEL_MIN: u8 = 1; + +// Best compression ratio +const GZIP_LEVEL_MAX: u8 = 9; + +impl GzipLevel { + pub const fn new() -> Self { + assert!(N >= GZIP_LEVEL_MIN, "Compression level too small"); + assert!(N <= GZIP_LEVEL_MAX, "Compression level too large"); + Self(N) + } +} + +#[derive( + Copy, + Clone, + Debug, + Default, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, +)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum CompressionAlgorithm { + // Selects a default compression algorithm. This is dependent on both the + // zpool and OS version. + On, + + // Disables compression. + #[default] + Off, + + // Selects the default Gzip compression level. + // + // According to the ZFS docs, this is "gzip-6", but that's a default value, + // which may change with OS updates. + Gzip, + + GzipN { + level: GzipLevel, + }, + Lz4, + Lzjb, + Zle, +} + +impl fmt::Display for CompressionAlgorithm { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use CompressionAlgorithm::*; + let s = match self { + On => "on", + Off => "off", + Gzip => "gzip", + GzipN { level } => { + return write!(f, "gzip-{}", level.0); + } + Lz4 => "lz4", + Lzjb => "lzjb", + Zle => "zle", + }; + write!(f, "{}", s) + } +} + +/// Configuration information necessary to request a single dataset +#[derive( + Clone, + Debug, + Deserialize, + Serialize, + JsonSchema, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, +)] +pub struct DatasetConfig { + /// The UUID of the dataset being requested + pub id: DatasetUuid, + + /// The dataset's name + pub name: DatasetName, + + /// The compression mode to be used by the dataset + pub compression: CompressionAlgorithm, + + /// The upper bound on the amount of storage used by this dataset + pub quota: Option, + + /// The lower bound on the amount of storage usable by this dataset + pub reservation: Option, +} + +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +pub struct DatasetsConfig { + /// generation number of this configuration + /// + /// This generation number is owned by the control plane (i.e., RSS or + /// Nexus, depending on whether RSS-to-Nexus handoff has happened). It + /// should not be bumped within Sled Agent. + /// + /// Sled Agent rejects attempts to set the configuration to a generation + /// older than the one it's currently running. + /// + /// Note that "Generation::new()", AKA, the first generation number, + /// is reserved for "no datasets". This is the default configuration + /// for a sled before any requests have been made. + pub generation: Generation, + + pub datasets: BTreeMap, +} + +impl Default for DatasetsConfig { + fn default() -> Self { + Self { generation: Generation::new(), datasets: BTreeMap::new() } + } +} + +impl Ledgerable for DatasetsConfig { + fn is_newer_than(&self, other: &Self) -> bool { + self.generation > other.generation + } + + // No need to do this, the generation number is provided externally. + fn generation_bump(&mut self) {} +} + +/// Identifies how a single dataset management operation may have succeeded or +/// failed. +#[derive(Debug, JsonSchema, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct DatasetManagementStatus { + pub dataset_name: DatasetName, + pub err: Option, +} + +/// The result from attempting to manage datasets. +#[derive(Default, Debug, JsonSchema, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[must_use = "this `DatasetManagementResult` may contain errors, which should be handled"] +pub struct DatasetsManagementResult { + pub status: Vec, +} + +impl DatasetsManagementResult { + pub fn has_error(&self) -> bool { + for status in &self.status { + if status.err.is_some() { + return true; + } + } + false + } +} + /// Uniquely identifies a disk. #[derive( Debug, diff --git a/common/src/lib.rs b/common/src/lib.rs index 6da32c56ba..b9d6dd3172 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -118,3 +118,27 @@ where async fn never_bail() -> Result { Ok(false) } + +/// A wrapper struct that does nothing other than elide the inner value from +/// [`std::fmt::Debug`] output. +/// +/// We define this within Omicron instead of using one of the many available +/// crates that do the same thing because it's trivial to do so, and we want the +/// flexibility to add traits to this type without needing to wait on upstream +/// to add an optional dependency. +/// +/// If you want to use this for secrets, consider that it might not do +/// everything you expect (it does not zeroize memory on drop, nor get in the +/// way of you removing the inner value from this wrapper struct). +#[derive( + Clone, Copy, serde::Deserialize, serde::Serialize, schemars::JsonSchema, +)] +#[repr(transparent)] +#[serde(transparent)] +pub struct NoDebug(pub T); + +impl std::fmt::Debug for NoDebug { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "..") + } +} diff --git a/common/src/policy.rs b/common/src/policy.rs index 677dbfe2b9..e615981a21 100644 --- a/common/src/policy.rs +++ b/common/src/policy.rs @@ -21,13 +21,13 @@ pub const COCKROACHDB_REDUNDANCY: usize = 5; /// The amount of redundancy for internal DNS servers. /// -/// Must be less than or equal to MAX_DNS_REDUNDANCY. -pub const DNS_REDUNDANCY: usize = 3; +/// Must be less than or equal to MAX_INTERNAL_DNS_REDUNDANCY. +pub const INTERNAL_DNS_REDUNDANCY: usize = 3; -/// The maximum amount of redundancy for DNS servers. +/// The maximum amount of redundancy for internal DNS servers. /// -/// This determines the number of addresses which are reserved for DNS servers. -pub const MAX_DNS_REDUNDANCY: usize = 5; +/// This determines the number of addresses which are reserved for internal DNS servers. +pub const MAX_INTERNAL_DNS_REDUNDANCY: usize = 5; /// The amount of redundancy for clickhouse servers /// diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 9ce4c66a80..48f5137698 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -246,7 +246,8 @@ impl DbUrlOptions { eprintln!("note: using database URL {}", &db_url); let db_config = db::Config { url: db_url.clone() }; - let pool = Arc::new(db::Pool::new(&log.clone(), &db_config)); + let pool = + Arc::new(db::Pool::new_single_host(&log.clone(), &db_config)); // Being a dev tool, we want to try this operation even if the schema // doesn't match what we expect. So we use `DataStore::new_unchecked()` @@ -4224,7 +4225,7 @@ async fn cmd_db_inventory( } async fn cmd_db_inventory_baseboard_ids( - conn: &DataStoreConnection<'_>, + conn: &DataStoreConnection, limit: NonZeroU32, ) -> Result<(), anyhow::Error> { #[derive(Tabled)] @@ -4261,7 +4262,7 @@ async fn cmd_db_inventory_baseboard_ids( } async fn cmd_db_inventory_cabooses( - conn: &DataStoreConnection<'_>, + conn: &DataStoreConnection, limit: NonZeroU32, ) -> Result<(), anyhow::Error> { #[derive(Tabled)] @@ -4302,7 +4303,7 @@ async fn cmd_db_inventory_cabooses( } async fn cmd_db_inventory_physical_disks( - conn: &DataStoreConnection<'_>, + conn: &DataStoreConnection, limit: NonZeroU32, args: InvPhysicalDisksArgs, ) -> Result<(), anyhow::Error> { @@ -4359,7 +4360,7 @@ async fn cmd_db_inventory_physical_disks( } async fn cmd_db_inventory_rot_pages( - conn: &DataStoreConnection<'_>, + conn: &DataStoreConnection, limit: NonZeroU32, ) -> Result<(), anyhow::Error> { #[derive(Tabled)] @@ -4394,7 +4395,7 @@ async fn cmd_db_inventory_rot_pages( } async fn cmd_db_inventory_collections_list( - conn: &DataStoreConnection<'_>, + conn: &DataStoreConnection, limit: NonZeroU32, ) -> Result<(), anyhow::Error> { #[derive(Tabled)] diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index ab5b83fe8e..58b32cb1f3 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -55,8 +55,9 @@ use tabled::settings::object::Columns; use tabled::settings::Padding; use tabled::Tabled; use tokio::sync::OnceCell; -use update_engine::display::StepIndexDisplay; +use update_engine::display::ProgressRatioDisplay; use update_engine::events::EventReport; +use update_engine::events::StepOutcome; use update_engine::EventBuffer; use update_engine::ExecutionStatus; use update_engine::ExecutionTerminalInfo; @@ -1681,7 +1682,7 @@ fn reason_str(reason: &ActivationReason) -> &'static str { } fn bgtask_apply_kv_style(table: &mut tabled::Table) { - let style = tabled::settings::Style::blank(); + let style = tabled::settings::Style::empty(); table.with(style).with( tabled::settings::Modify::new(Columns::first()) // Background task tables are offset by 4 characters. @@ -1724,38 +1725,7 @@ fn push_event_buffer_summary( ) { match event_buffer { Ok(buffer) => { - let Some(summary) = buffer.root_execution_summary() else { - builder.push_record(["status:", "(no information found)"]); - return; - }; - - match summary.execution_status { - ExecutionStatus::NotStarted => { - builder.push_record(["status:", "not started"]); - } - ExecutionStatus::Running { step_key, .. } => { - let step_data = buffer.get(&step_key).expect("step exists"); - builder.push_record([ - "status:".to_string(), - format!( - "running: {} (step {})", - step_data.step_info().description, - StepIndexDisplay::new( - step_key.index + 1, - summary.total_steps - ), - ), - ]); - } - ExecutionStatus::Terminal(info) => { - push_event_buffer_terminal_info( - &info, - summary.total_steps, - &buffer, - builder, - ); - } - } + event_buffer_summary_impl(buffer, builder); } Err(error) => { builder.push_record([ @@ -1772,6 +1742,64 @@ fn push_event_buffer_summary( } } +fn event_buffer_summary_impl( + buffer: EventBuffer, + builder: &mut tabled::builder::Builder, +) { + let Some(summary) = buffer.root_execution_summary() else { + builder.push_record(["status:", "(no information found)"]); + return; + }; + + match summary.execution_status { + ExecutionStatus::NotStarted => { + builder.push_record(["status:", "not started"]); + } + ExecutionStatus::Running { step_key, .. } => { + let step_data = buffer.get(&step_key).expect("step exists"); + builder.push_record([ + "status:".to_string(), + format!( + "running: {} (step {})", + step_data.step_info().description, + ProgressRatioDisplay::index_and_total( + step_key.index, + summary.total_steps, + ), + ), + ]); + } + ExecutionStatus::Terminal(info) => { + push_event_buffer_terminal_info( + &info, + summary.total_steps, + &buffer, + builder, + ); + } + } + + // Also look for warnings. + for (_, step_data) in buffer.iter_steps_recursive() { + if let Some(reason) = step_data.step_status().completion_reason() { + if let Some(info) = reason.step_completed_info() { + if let StepOutcome::Warning { message, .. } = &info.outcome { + builder.push_record([ + "warning:".to_string(), + // This can be a nested step, so don't print out the + // index. + format!( + "at: {}: {}", + step_data.step_info().description, + message + ), + ]); + } + } + } + } +} + fn push_event_buffer_terminal_info( info: &ExecutionTerminalInfo, total_steps: usize, @@ -1789,7 +1817,10 @@ fn push_event_buffer_terminal_info( let v = format!( "failed at: {} (step {})", step_data.step_info().description, - StepIndexDisplay::new(info.step_key.index, total_steps) + ProgressRatioDisplay::index_and_total( + info.step_key.index, + total_steps, + ) ); builder.push_record(["status:".to_string(), v]); @@ -1800,7 +1831,10 @@ fn push_event_buffer_terminal_info( let v = format!( "aborted at: {} (step {})", step_data.step_info().description, - StepIndexDisplay::new(info.step_key.index, total_steps) + ProgressRatioDisplay::index_and_total( + info.step_key.index, + total_steps, + ) ); builder.push_record(["status:".to_string(), v]); diff --git a/dev-tools/openapi-manager/Cargo.toml b/dev-tools/openapi-manager/Cargo.toml index 2ca1bc3e4d..211e134016 100644 --- a/dev-tools/openapi-manager/Cargo.toml +++ b/dev-tools/openapi-manager/Cargo.toml @@ -21,6 +21,7 @@ fs-err.workspace = true gateway-api.workspace = true indent_write.workspace = true installinator-api.workspace = true +nexus-external-api.workspace = true nexus-internal-api.workspace = true omicron-workspace-hack.workspace = true openapiv3.workspace = true diff --git a/dev-tools/openapi-manager/README.adoc b/dev-tools/openapi-manager/README.adoc index 1aadaa2c0c..e6b28b44f6 100644 --- a/dev-tools/openapi-manager/README.adoc +++ b/dev-tools/openapi-manager/README.adoc @@ -4,19 +4,15 @@ This tool manages the OpenAPI documents (JSON files) checked into Omicron's `ope NOTE: For more information about API traits, see https://rfd.shared.oxide.computer/rfd/0479[RFD 479]. -Currently, a subset of OpenAPI documents is managed by this tool. Eventually, all of the OpenAPI documents in Omicron will be managed by this tool; work to make that happen is ongoing. - -To check whether your document is managed, run `cargo xtask openapi list`: it will list out all managed OpenAPI documents. If your document is not on the list, it is unmanaged. - == Basic usage The OpenAPI manager is meant to be invoked via `cargo xtask openapi`. Currently, three commands are provided: -* `cargo xtask openapi list`: List information about currently-managed documents. -* `cargo xtask openapi check`: Check that all of the managed documents are up-to-date. +* `cargo xtask openapi list`: List information about OpenAPI documents. +* `cargo xtask openapi check`: Check that all of the documents are up-to-date. * `cargo xtask openapi generate`: Update and generate OpenAPI documents. -There is also a test which makes sure that all managed documents are up-to-date, and tells you to run `cargo xtask openapi generate` if they aren't. +There is also a test which makes sure that all documents are up-to-date, and tells you to run `cargo xtask openapi generate` if they aren't. === API crates [[api_crates]] @@ -49,40 +45,13 @@ In the implementation crate: . Add a dependency on the API crate. . Following the example in https://rfd.shared.oxide.computer/rfd/0479#guide_api_implementation[RFD 479's _API implementation_], provide an implementation of the trait. -Once the API crate is defined, perform the steps in <> below. - -=== Converting existing documents - -Existing, unmanaged documents are generated via *function-based servers*: a set of functions that some code combines into a Dropshot `ApiDescription`. (There is also likely an expectorate test which ensures that the document is up-to-date.) - -The first step is to convert the function-based server into an API trait. To do so, create an API crate (see <> above). - -. Add the API crate to the workspace's `Cargo.toml`: `members` and `default-members`, and a reference in `[workspace.dependencies]`. -. Follow the instructions in https://rfd.shared.oxide.computer/rfd/0479#guide_converting_functions_to_traits[RFD 479's _Converting functions to API traits_] for the API crate. - -In the implementation crate: - -. Continue following the instructions in https://rfd.shared.oxide.computer/rfd/0479#guide_converting_functions_to_traits[RFD 479's _Converting functions to API traits_] for where the endpoint functions are currently defined. -. Find the test which currently manages the document (try searching the repo for `openapi_lint::validate`). If it performs any checks on the document beyond `openapi_lint::validate` or `openapi_lint::validate_external`, see <>. - -Next, perform the steps in <> below. - -Finally, remove: - -. The test which used to manage the document. The OpenAPI manager includes a test that will automatically run in CI. -. The binary subcommand (typically called `openapi`) that generated the OpenAPI document. The test was the only practical use of this subcommand. - -=== Adding the API crate to the manager [[add_to_manager]] - Once the API crate is defined, inform the OpenAPI manager of its existence. Within this directory: . In `Cargo.toml`, add a dependency on the API crate. . In `src/spec.rs`, add the crate to the `all_apis` function. (Please keep the list sorted by filename.) -To ensure everything works well, run `cargo xtask openapi generate`. - -* Your OpenAPI document should be generated on disk and listed in the output. -* If you're converting an existing API, the only changes should be the ones you might have introduced as part of the refactor. If there are significant changes, something's gone wrong--maybe you missed an endpoint? +To ensure everything works well, run `cargo xtask openapi generate`. Your +OpenAPI document should be generated on disk and listed in the output. ==== Performing extra validation [[extra_validation]] @@ -90,10 +59,19 @@ By default, the OpenAPI manager does basic validation on the generated document. It's best to put extra validation next to the trait, within the API crate. -. In the API crate, add dependencies on `anyhow` and `openapiv3`. -. Define a function with signature `fn extra_validation(openapi: &openapiv3::OpenAPI) -> anyhow::Result<()>` which performs the extra validation steps. +. In the API crate, add dependencies on `openapiv3` and `openapi-manager-types`. +. Define a function with signature `fn validate_api(spec: &openapiv3::OpenAPI, mut cx: openapi_manager_types::ValidationContext<'_>) which performs the extra validation steps. . In `all_apis`, set the `extra_validation` field to this function. +Currently, the validator can do two things: + +. Via the `ValidationContext::report_error` function, report validation errors. +. Via the `ValidationContext::record_file_contents` function, assert the contents of other generated files. + +(This can be made richer as needed.) + +For an example, see `validate_api` in the `nexus-external-api` crate. + == Design notes The OpenAPI manager uses the new support for Dropshot API traits described in https://rfd.shared.oxide.computer/rfd/0479[RFD 479]. diff --git a/dev-tools/openapi-manager/src/spec.rs b/dev-tools/openapi-manager/src/spec.rs index e74cf7ed7a..03511a7945 100644 --- a/dev-tools/openapi-manager/src/spec.rs +++ b/dev-tools/openapi-manager/src/spec.rs @@ -79,6 +79,16 @@ pub fn all_apis() -> Vec { filename: "installinator.json", extra_validation: None, }, + ApiSpec { + title: "Oxide Region API", + version: "20240821.0", + description: "API for interacting with the Oxide control plane", + boundary: ApiBoundary::External, + api_description: + nexus_external_api::nexus_external_api_mod::stub_api_description, + filename: "nexus.json", + extra_validation: Some(nexus_external_api::validate_api), + }, ApiSpec { title: "Nexus internal API", version: "0.0.1", diff --git a/dev-tools/xtask/Cargo.toml b/dev-tools/xtask/Cargo.toml index ec1b7825c6..508d0c73ee 100644 --- a/dev-tools/xtask/Cargo.toml +++ b/dev-tools/xtask/Cargo.toml @@ -24,6 +24,7 @@ workspace = true # downstream binaries do depend on it.) anyhow.workspace = true camino.workspace = true +camino-tempfile.workspace = true cargo_toml = "0.20" cargo_metadata.workspace = true clap.workspace = true @@ -32,5 +33,6 @@ macaddr.workspace = true serde.workspace = true swrite.workspace = true tabled.workspace = true +textwrap.workspace = true toml.workspace = true usdt.workspace = true diff --git a/dev-tools/xtask/src/check_workspace_deps.rs b/dev-tools/xtask/src/check_workspace_deps.rs index 73d5643ffb..a9627569b9 100644 --- a/dev-tools/xtask/src/check_workspace_deps.rs +++ b/dev-tools/xtask/src/check_workspace_deps.rs @@ -125,9 +125,6 @@ pub fn run_cmd() -> Result<()> { // Including xtask causes hakari to not work as well and build // times to be longer (omicron#4392). "xtask", - // The tests here should not be run by default, as they require - // a running control plane. - "end-to-end-tests", ] .contains(&package.name.as_str()) .then_some(&package.id) diff --git a/dev-tools/xtask/src/clippy.rs b/dev-tools/xtask/src/clippy.rs index 7924a05574..229d0e126e 100644 --- a/dev-tools/xtask/src/clippy.rs +++ b/dev-tools/xtask/src/clippy.rs @@ -4,7 +4,8 @@ //! Subcommand: cargo xtask clippy -use anyhow::{bail, Context, Result}; +use crate::common::run_subcmd; +use anyhow::Result; use clap::Parser; use std::process::Command; @@ -51,25 +52,5 @@ pub fn run_cmd(args: ClippyArgs) -> Result<()> { .arg("--deny") .arg("warnings"); - eprintln!( - "running: {:?} {}", - &cargo, - command - .get_args() - .map(|arg| format!("{:?}", arg.to_str().unwrap())) - .collect::>() - .join(" ") - ); - - let exit_status = command - .spawn() - .context("failed to spawn child process")? - .wait() - .context("failed to wait for child process")?; - - if !exit_status.success() { - bail!("clippy failed: {}", exit_status); - } - - Ok(()) + run_subcmd(command) } diff --git a/dev-tools/xtask/src/common.rs b/dev-tools/xtask/src/common.rs new file mode 100644 index 0000000000..03b17a560f --- /dev/null +++ b/dev-tools/xtask/src/common.rs @@ -0,0 +1,34 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Common xtask command helpers + +use anyhow::{bail, Context, Result}; +use std::process::Command; + +/// Runs the given command, printing some basic debug information around it, and +/// failing with an error message if the command does not exit successfully +pub fn run_subcmd(mut command: Command) -> Result<()> { + eprintln!( + "running: {} {}", + command.get_program().to_str().unwrap(), + command + .get_args() + .map(|arg| format!("{:?}", arg.to_str().unwrap())) + .collect::>() + .join(" ") + ); + + let exit_status = command + .spawn() + .context("failed to spawn child process")? + .wait() + .context("failed to wait for child process")?; + + if !exit_status.success() { + bail!("failed: {}", exit_status); + } + + Ok(()) +} diff --git a/dev-tools/xtask/src/live_tests.rs b/dev-tools/xtask/src/live_tests.rs new file mode 100644 index 0000000000..e63881a1fd --- /dev/null +++ b/dev-tools/xtask/src/live_tests.rs @@ -0,0 +1,159 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Subcommand: cargo xtask live-tests + +use crate::common::run_subcmd; +use anyhow::{bail, Context, Result}; +use clap::Parser; +use std::process::Command; + +#[derive(Parser)] +pub struct Args {} + +pub fn run_cmd(_args: Args) -> Result<()> { + const NAME: &str = "live-tests-archive"; + + // The live tests operate in deployed environments, which always run + // illumos. Bail out quickly if someone tries to run this on a system whose + // binaries won't be usable. (We could compile this subcommand out + // altogether on non-illumos systems, but it seems more confusing to be + // silently missing something you might expect to be there. Plus, you can + // still check and even build *this* code on non-illumos systems.) + if cfg!(not(target_os = "illumos")) { + bail!("live-tests archive can only be built on illumos systems"); + } + + let tmpdir_root = + camino_tempfile::tempdir().context("creating temporary directory")?; + let final_tarball = camino::Utf8PathBuf::try_from( + std::env::current_dir() + .map(|d| d.join("target")) + .context("getting current directory")?, + ) + .context("non-UTF-8 current directory")? + .join(format!("{}.tgz", NAME)); + let proto_root = tmpdir_root.path().join(NAME); + let nextest_archive_file = proto_root.join("omicron-live-tests.tar.zst"); + + eprintln!("using temporary directory: {}", tmpdir_root.path()); + eprintln!("will create archive file: {}", nextest_archive_file); + eprintln!("output tarball: {}", final_tarball); + eprintln!(); + + std::fs::create_dir(&proto_root) + .with_context(|| format!("mkdir {:?}", &proto_root))?; + + let cargo = + std::env::var("CARGO").unwrap_or_else(|_| String::from("cargo")); + let mut command = Command::new(&cargo); + + command.arg("nextest"); + command.arg("archive"); + command.arg("--package"); + command.arg("omicron-live-tests"); + command.arg("--archive-file"); + command.arg(&nextest_archive_file); + run_subcmd(command)?; + + // Using nextest archives requires that the source be separately transmitted + // to the system where the tests will be run. We're trying to automate + // that. So let's bundle up the source and the nextest archive into one big + // tarball. But which source files do we bundle? We need: + // + // - Cargo.toml (nextest expects to find this) + // - .config/nextest.toml (nextest's configuration, which is used while + // running the tests) + // - live-tests (this is where the tests live, and they might expect stuff + // that exists in here like expectorate files) + // + // plus the nextext archive file. + // + // To avoid creating a tarbomb, we want all the files prefixed with + // "live-tests-archive/". There's no great way to do this with the illumos + // tar(1) except to create a temporary directory called "live-tests-archive" + // that contains the files and then tar'ing up that. + // + // Ironically, an easy way to construct that directory is with tar(1). + let mut command = Command::new("bash"); + command.arg("-c"); + command.arg(format!( + "tar cf - Cargo.toml .config/nextest.toml live-tests | \ + tar xf - -C {:?}", + &proto_root + )); + run_subcmd(command)?; + + let mut command = Command::new("tar"); + command.arg("cf"); + command.arg(&final_tarball); + command.arg("-C"); + command.arg(tmpdir_root.path()); + command.arg(NAME); + run_subcmd(command)?; + + drop(tmpdir_root); + + eprint!("created: "); + println!("{}", &final_tarball); + eprintln!("\nTo use this:\n"); + eprintln!( + "1. Copy the tarball to the switch zone in a deployed Omicron system.\n" + ); + let raw = &[ + "scp \\", + &format!("{} \\", &final_tarball), + "root@YOUR_SCRIMLET_GZ_IP:/zone/oxz_switch/root/root", + ] + .join("\n"); + let text = textwrap::wrap( + &raw, + textwrap::Options::new(160) + .initial_indent(" e.g., ") + .subsequent_indent(" "), + ); + eprintln!("{}\n", text.join("\n")); + eprintln!("2. Copy the `cargo-nextest` binary to the same place.\n"); + let raw = &[ + "scp \\", + "$(which cargo-nextest) \\", + "root@YOUR_SCRIMLET_GZ_IP:/zone/oxz_switch/root/root", + ] + .join("\n"); + let text = textwrap::wrap( + &raw, + textwrap::Options::new(160) + .initial_indent(" e.g., ") + .subsequent_indent(" "), + ); + eprintln!("{}\n", text.join("\n")); + eprintln!("3. On that system, unpack the tarball with:\n"); + eprintln!(" tar xzf {}\n", final_tarball.file_name().unwrap()); + eprintln!("4. On that system, run tests with:\n"); + // TMPDIR=/var/tmp puts stuff on disk, cached as needed, rather than the + // default /tmp which requires that stuff be in-memory. That can lead to + // great sadness if the tests wind up writing a lot of data. + // + // nextest configuration for these tests is specified in the "live-tests" + // profile. + let raw = &[ + "TMPDIR=/var/tmp ./cargo-nextest nextest run --profile=live-tests \\", + &format!( + "--archive-file {}/{} \\", + NAME, + nextest_archive_file.file_name().unwrap() + ), + &format!("--workspace-remap {}", NAME), + ] + .join("\n"); + let text = textwrap::wrap( + &raw, + textwrap::Options::new(160) + .initial_indent(" ") + .subsequent_indent(" "), + ); + eprintln!("{}\n", text.join("\n")); + + Ok(()) +} diff --git a/dev-tools/xtask/src/main.rs b/dev-tools/xtask/src/main.rs index 02fd05a198..9880adeb67 100644 --- a/dev-tools/xtask/src/main.rs +++ b/dev-tools/xtask/src/main.rs @@ -16,8 +16,10 @@ use std::process::Command; mod check_features; mod check_workspace_deps; mod clippy; +mod common; #[cfg_attr(not(target_os = "illumos"), allow(dead_code))] mod external; +mod live_tests; mod usdt; #[cfg(target_os = "illumos")] @@ -59,6 +61,9 @@ enum Cmds { /// Download binaries, OpenAPI specs, and other out-of-repo utilities. Download(external::External), + /// Create a bundle of live tests + LiveTests(live_tests::Args), + /// Utilities for working with MGS. MgsDev(external::External), /// Utilities for working with Omicron. @@ -127,6 +132,7 @@ fn main() -> Result<()> { external.exec_bin("xtask-downloader") } } + Cmds::LiveTests(args) => live_tests::run_cmd(args), Cmds::MgsDev(external) => external.exec_bin("mgs-dev"), Cmds::OmicronDev(external) => external.exec_bin("omicron-dev"), Cmds::Openapi(external) => external.exec_bin("openapi-manager"), diff --git a/dev-tools/xtask/src/virtual_hardware.rs b/dev-tools/xtask/src/virtual_hardware.rs index d28c3d9037..29738016f5 100644 --- a/dev-tools/xtask/src/virtual_hardware.rs +++ b/dev-tools/xtask/src/virtual_hardware.rs @@ -49,6 +49,10 @@ enum Commands { #[clap(long, default_value = PXA_MAC_DEFAULT)] pxa_mac: String, + + /// Size in bytes for created vdevs + #[clap(long, default_value_t = 20 * GB)] + vdev_size: u64, }, /// Destroy virtual hardware which was initialized with "Create" Destroy, @@ -96,7 +100,6 @@ pub struct Args { static NO_INSTALL_MARKER: &'static str = "/etc/opt/oxide/NO_INSTALL"; const GB: u64 = 1 << 30; -const VDEV_SIZE: u64 = 20 * GB; const ARP: &'static str = "/usr/sbin/arp"; const DLADM: &'static str = "/usr/sbin/dladm"; @@ -163,6 +166,7 @@ pub fn run_cmd(args: Args) -> Result<()> { gateway_mac, pxa, pxa_mac, + vdev_size, } => { let physical_link = if let Some(l) = physical_link { l @@ -172,7 +176,7 @@ pub fn run_cmd(args: Args) -> Result<()> { println!("creating virtual hardware"); if matches!(args.scope, Scope::All | Scope::Disks) { - ensure_vdevs(&sled_agent_config, &args.vdev_dir)?; + ensure_vdevs(&sled_agent_config, &args.vdev_dir, vdev_size)?; } if matches!(args.scope, Scope::All | Scope::Network) && softnpu_mode == "zone" @@ -503,6 +507,7 @@ impl SledAgentConfig { fn ensure_vdevs( sled_agent_config: &Utf8Path, vdev_dir: &Utf8Path, + vdev_size: u64, ) -> Result<()> { let config = SledAgentConfig::read(sled_agent_config)?; @@ -522,7 +527,7 @@ fn ensure_vdevs( } else { println!("creating {vdev_path}"); let file = std::fs::File::create(&vdev_path)?; - file.set_len(VDEV_SIZE)?; + file.set_len(vdev_size)?; } } Ok(()) diff --git a/docs/adding-an-endpoint.adoc b/docs/adding-an-endpoint.adoc index cebaae4c52..d9e5c559b4 100644 --- a/docs/adding-an-endpoint.adoc +++ b/docs/adding-an-endpoint.adoc @@ -12,17 +12,21 @@ NOTE: This guide is not intended to be exhaustive, or even particularly detailed. For that, refer to the documentation which exists in the codebase -- this document should act as a jumping-off point. -=== **HTTP** -* Add endpoints for either the internal or external API -** xref:../nexus/src/external_api/http_entrypoints.rs[The External API] is customer-facing, and provides interfaces for both developers and operators -** xref:../nexus/src/internal_api/http_entrypoints.rs[The Internal API] is internal, and provides interfaces for services on the Oxide rack (such as the Sled Agent) to call -** Register endpoints in the `register_endpoints` method (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/src/external_api/http_entrypoints.rs#L84[Example]) +== **HTTP** + +* Add endpoint _definitions_ for either the internal or external API +** xref:../nexus/external-api/src/lib.rs[The External API] is customer-facing, and provides interfaces for both developers and operators +** xref:../nexus/internal-api/src/lib.rs[The Internal API] is internal, and provides interfaces for services on the Oxide rack (such as the Sled Agent) to call +* Add the corresponding _implementations_ to the respective `http_entrypoints.rs` files: +** xref:../nexus/src/external_api/http_entrypoints.rs[The External API's `http_entrypoints.rs`] +** xref:../nexus/src/internal_api/http_entrypoints.rs[The Internal API's `http_entrypoints.rs`] ** These endpoints typically call into the *Application* layer, and do not access the database directly * Inputs and Outputs ** Input parameters are defined in https://github.com/oxidecomputer/omicron/blob/main/nexus/types/src/external_api/params.rs[params.rs] (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/types/src/external_api/params.rs#L587-L601[Example]) ** Output views are defined in https://github.com/oxidecomputer/omicron/blob/main/nexus/types/src/external_api/views.rs[views.rs] (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/types/src/external_api/views.rs#L270-L274[Example]) -=== **Lookup & Authorization** +== **Lookup & Authorization** + * Declare a new resource-to-be-looked-up via `lookup_resource!` in xref:../nexus/src/db/lookup.rs[lookup.rs] (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/src/db/lookup.rs#L557-L564[Example]) ** This defines a new struct named after your resource, with some https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/db-macros/src/lookup.rs#L521-L628[auto-generated methods], including `lookup_for` (look up the authz object), `fetch_for` (look up and return the object), and more * Add helper functions to `LookupPath` to make it possible to fetch the resource by either UUID or name (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/src/db/lookup.rs#L225-L237[Example]) @@ -32,12 +36,14 @@ this document should act as a jumping-off point. ** If you define `polar_snippet = Custom`, you should edit the omicron.polar file to describe the authorization policy for your object (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/src/authz/omicron.polar#L376-L393[Example]) * Either way, you should add reference the new resource when https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/src/authz/oso_generic.rs#L119-L148[constructing the Oso structure] -=== **Application** +== **Application** + * Add any "business logic" for the resource to xref:../nexus/src/app[the app directory] * This layer bridges the gap between the database and external services. * If your application logic involes any multi-step operations which would be interrupted by Nexus stopping mid-execution (due to reboot, crash, failure, etc), it is recommended to use a https://github.com/oxidecomputer/omicron/tree/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/src/app/sagas[saga] to define the operations durably. -=== **Database** +== **Database** + * `CREATE TABLE` for the resource in xref:../schema/crdb/dbinit.sql[dbinit.sql] (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/common/src/sql/dbinit.sql#L1103-L1129[Example]) * Add an equivalent schema for the resource in xref:../nexus/db-model/src/schema.rs[schema.rs], which allows https://docs.diesel.rs/master/diesel/index.html[Diesel] to translate raw SQL to rust queries (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/db-model/src/schema.rs#L144-L155[Example]) * Add a Rust representation of the database object to xref:../nexus/db-model/src[the DB model] (https://github.com/oxidecomputer/omicron/blob/1dfe47c1b3122bc4f32a9c517cb31b1600581ea2/nexus/db-model/src/ip_pool.rs#L24-L40[Example]) @@ -48,22 +54,10 @@ this document should act as a jumping-off point. * Authorization ** There exists a https://github.com/oxidecomputer/omicron/blob/main/nexus/src/authz/policy_test[policy test] which compares all Oso objects against an expected policy. New resources are usually added to https://github.com/oxidecomputer/omicron/blob/main/nexus/src/authz/policy_test/resources.rs[resources.rs] to get coverage. -* openapi -** Nexus generates a new openapi spec from the dropshot endpoints. If you modify endpoints, you'll need to update openapi JSON files. -*** The following commands may be used to update APIs: -+ -[source, rust] ----- -$ cargo run -p omicron-nexus --bin nexus -- -I nexus/examples/config.toml > openapi/nexus-internal.json -$ cargo run -p omicron-nexus --bin nexus -- -O nexus/examples/config.toml > openapi/nexus.json -$ cargo run -p omicron-sled-agent --bin sled-agent -- openapi > openapi/sled-agent.json ----- -*** Alternative, you can run: -+ -[source, rust] ----- -$ EXPECTORATE=overwrite cargo test_nexus_openapi test_nexus_openapi_internal test_sled_agent_openapi_sled ----- +* OpenAPI +** Once you've added or changed endpoint definitions in `nexus-external-api` or `nexus-internal-api`, you'll need to update the corresponding OpenAPI documents (the JSON files in `openapi/`). +** To update all OpenAPI documents, run `cargo xtask openapi generate`. +** This does not require you to provide an implementation, or to get either omicron-nexus or omicron-sled-agent to compile: just the definition in the API crate is sufficient. * Integration Tests ** Nexus' https://github.com/oxidecomputer/omicron/tree/main/nexus/tests/integration_tests[integration tests] are used to cross the HTTP interface for testing. Typically, one file is used "per-resource". *** These tests use a simulated Sled Agent, and keep the "Nexus" object in-process, so it can still be accessed and modified for invasive testing. diff --git a/docs/crdb-upgrades.adoc b/docs/crdb-upgrades.adoc index 52231ee199..6613345465 100644 --- a/docs/crdb-upgrades.adoc +++ b/docs/crdb-upgrades.adoc @@ -64,7 +64,7 @@ a tick, but they must occur in that order.) of CockroachDB versions: + .... -EXPECTORATE=overwrite cargo nextest run -p omicron-nexus -- integration_tests::commands::test_nexus_openapi_internal +cargo xtask openapi generate .... . Run the full test suite, which should catch any unexpected SQL compatibility issues between releases and help validate that your diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml index b2400f7603..044f70ef23 100644 --- a/end-to-end-tests/Cargo.toml +++ b/end-to-end-tests/Cargo.toml @@ -19,8 +19,8 @@ omicron-test-utils.workspace = true oxide-client.workspace = true rand.workspace = true reqwest = { workspace = true, features = ["cookies"] } -russh = "0.44.1" -russh-keys = "0.44.0" +russh = "0.45.0" +russh-keys = "0.45.0" serde.workspace = true serde_json.workspace = true sled-agent-types.workspace = true diff --git a/end-to-end-tests/README.adoc b/end-to-end-tests/README.adoc index b9766db809..3e31f2b382 100644 --- a/end-to-end-tests/README.adoc +++ b/end-to-end-tests/README.adoc @@ -4,6 +4,8 @@ These tests run in Buildomat. They are built by the xref:../.github/buildomat/jo This package is not built or run by default (it is excluded from `default-members` in xref:../Cargo.toml[]). +See also: xref:../live-tests/README.adoc[omicron-live-tests]. + == Running these tests on your machine 1. xref:../docs/how-to-run.adoc[Make yourself a Gimlet]. diff --git a/gateway/src/context.rs b/gateway/src/context.rs index 15592145cf..dc5717604b 100644 --- a/gateway/src/context.rs +++ b/gateway/src/context.rs @@ -39,16 +39,18 @@ impl ServerContext { OnceLock::new() }; - const START_LATENCY_DECADE: i16 = -6; - const END_LATENCY_DECADE: i16 = 3; + // Track from 1 microsecond == 1e3 nanoseconds + const LATENCY_START_POWER: u16 = 3; + // To 1000s == 1e9 * 1e3 == 1e12 nanoseconds + const LATENCY_END_POWER: u16 = 12; let latencies = - oximeter_instruments::http::LatencyTracker::with_latency_decades( + oximeter_instruments::http::LatencyTracker::with_log_linear_bins( oximeter_instruments::http::HttpService { name: "management-gateway-service".into(), id, }, - START_LATENCY_DECADE, - END_LATENCY_DECADE, + LATENCY_START_POWER, + LATENCY_END_POWER, ) .expect("start and end decades are hardcoded and should be valid"); diff --git a/illumos-utils/src/zfs.rs b/illumos-utils/src/zfs.rs index 139e6fe607..5d512677f8 100644 --- a/illumos-utils/src/zfs.rs +++ b/illumos-utils/src/zfs.rs @@ -6,6 +6,7 @@ use crate::{execute, PFEXEC}; use camino::{Utf8Path, Utf8PathBuf}; +use omicron_common::disk::CompressionAlgorithm; use omicron_common::disk::DiskIdentity; use std::fmt; @@ -203,7 +204,8 @@ pub struct EncryptionDetails { #[derive(Debug, Default)] pub struct SizeDetails { pub quota: Option, - pub compression: Option<&'static str>, + pub reservation: Option, + pub compression: CompressionAlgorithm, } #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))] @@ -259,9 +261,27 @@ impl Zfs { Ok(()) } - /// Creates a new ZFS filesystem named `name`, unless one already exists. + /// Creates a new ZFS filesystem unless one already exists. /// - /// Applies an optional quota, provided _in bytes_. + /// - `name`: the full path to the zfs dataset + /// - `mountpoint`: The expected mountpoint of this filesystem. + /// If the filesystem already exists, and is not mounted here, and error is + /// returned. + /// - `zoned`: identifies whether or not this filesystem should be + /// used in a zone. Only used when creating a new filesystem - ignored + /// if the filesystem already exists. + /// - `do_format`: if "false", prevents a new filesystem from being created, + /// and returns an error if it is not found. + /// - `encryption_details`: Ensures a filesystem as an encryption root. + /// For new filesystems, this supplies the key, and all datasets within this + /// root are implicitly encrypted. For existing filesystems, ensures that + /// they are mounted (and that keys are loaded), but does not verify the + /// input details. + /// - `size_details`: If supplied, sets size-related information. These + /// values are set on both new filesystem creation as well as when loading + /// existing filesystems. + /// - `additional_options`: Additional ZFS options, which are only set when + /// creating new filesystems. #[allow(clippy::too_many_arguments)] pub fn ensure_filesystem( name: &str, @@ -274,10 +294,18 @@ impl Zfs { ) -> Result<(), EnsureFilesystemError> { let (exists, mounted) = Self::dataset_exists(name, &mountpoint)?; if exists { - if let Some(SizeDetails { quota, compression }) = size_details { + if let Some(SizeDetails { quota, reservation, compression }) = + size_details + { // apply quota and compression mode (in case they've changed across // sled-agent versions since creation) - Self::apply_properties(name, &mountpoint, quota, compression)?; + Self::apply_properties( + name, + &mountpoint, + quota, + reservation, + compression, + )?; } if encryption_details.is_none() { @@ -351,42 +379,64 @@ impl Zfs { })?; } - if let Some(SizeDetails { quota, compression }) = size_details { + if let Some(SizeDetails { quota, reservation, compression }) = + size_details + { // Apply any quota and compression mode. - Self::apply_properties(name, &mountpoint, quota, compression)?; + Self::apply_properties( + name, + &mountpoint, + quota, + reservation, + compression, + )?; } Ok(()) } + /// Applies the following properties to the filesystem. + /// + /// If any of the options are not supplied, a default "none" or "off" + /// value is supplied. fn apply_properties( name: &str, mountpoint: &Mountpoint, quota: Option, - compression: Option<&'static str>, + reservation: Option, + compression: CompressionAlgorithm, ) -> Result<(), EnsureFilesystemError> { - if let Some(quota) = quota { - if let Err(err) = - Self::set_value(name, "quota", &format!("{quota}")) - { - return Err(EnsureFilesystemError { - name: name.to_string(), - mountpoint: mountpoint.clone(), - // Take the execution error from the SetValueError - err: err.err.into(), - }); - } + let quota = quota + .map(|q| q.to_string()) + .unwrap_or_else(|| String::from("none")); + let reservation = reservation + .map(|r| r.to_string()) + .unwrap_or_else(|| String::from("none")); + let compression = compression.to_string(); + + if let Err(err) = Self::set_value(name, "quota", "a) { + return Err(EnsureFilesystemError { + name: name.to_string(), + mountpoint: mountpoint.clone(), + // Take the execution error from the SetValueError + err: err.err.into(), + }); } - if let Some(compression) = compression { - if let Err(err) = Self::set_value(name, "compression", compression) - { - return Err(EnsureFilesystemError { - name: name.to_string(), - mountpoint: mountpoint.clone(), - // Take the execution error from the SetValueError - err: err.err.into(), - }); - } + if let Err(err) = Self::set_value(name, "reservation", &reservation) { + return Err(EnsureFilesystemError { + name: name.to_string(), + mountpoint: mountpoint.clone(), + // Take the execution error from the SetValueError + err: err.err.into(), + }); + } + if let Err(err) = Self::set_value(name, "compression", &compression) { + return Err(EnsureFilesystemError { + name: name.to_string(), + mountpoint: mountpoint.clone(), + // Take the execution error from the SetValueError + err: err.err.into(), + }); } Ok(()) } diff --git a/ipcc/Cargo.toml b/ipcc/Cargo.toml index a9278349e1..cfde3f737a 100644 --- a/ipcc/Cargo.toml +++ b/ipcc/Cargo.toml @@ -9,13 +9,12 @@ workspace = true [dependencies] ciborium.workspace = true -libc.workspace = true omicron-common.workspace = true serde.workspace = true thiserror.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true -cfg-if.workspace = true +libipcc.workspace = true [dev-dependencies] omicron-common = { workspace = true, features = ["testing"] } diff --git a/ipcc/build.rs b/ipcc/build.rs deleted file mode 100644 index a64133dac2..0000000000 --- a/ipcc/build.rs +++ /dev/null @@ -1,16 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -/// This path is where Oxide specific libraries live on helios systems. -#[cfg(target_os = "illumos")] -static OXIDE_PLATFORM: &str = "/usr/platform/oxide/lib/amd64/"; - -fn main() { - println!("cargo:rerun-if-changed=build.rs"); - #[cfg(target_os = "illumos")] - { - println!("cargo:rustc-link-arg=-Wl,-R{}", OXIDE_PLATFORM); - println!("cargo:rustc-link-search={}", OXIDE_PLATFORM); - } -} diff --git a/ipcc/src/ffi.rs b/ipcc/src/ffi.rs deleted file mode 100644 index 420c1ddcde..0000000000 --- a/ipcc/src/ffi.rs +++ /dev/null @@ -1,83 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// Copyright 2023 Oxide Computer Company - -#![allow(non_upper_case_globals)] -#![allow(non_camel_case_types)] -#![allow(non_snake_case)] - -use std::ffi::{c_char, c_int, c_uint}; - -/// Opaque libipcc handle -#[repr(C)] -pub(crate) struct libipcc_handle_t { - _data: [u8; 0], - _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, -} - -/// Indicates that there was no error. Used as the initialized value when -/// calling into libipcc. -pub(crate) const LIBIPCC_ERR_OK: libipcc_err_t = 0; - -/// Indicates that there was a memory allocation error. The system error -/// contains the specific errno. -pub(crate) const LIBIPCC_ERR_NO_MEM: libipcc_err_t = 1; - -/// One of the function parameters does not pass validation. There will be more -/// detail available via libipcc_errmsg(). -pub(crate) const LIBIPCC_ERR_INVALID_PARAM: libipcc_err_t = 2; - -/// An internal error occurred. There will be more detail available via -/// libipcc_errmsg() and libipcc_syserr(). -pub(crate) const LIBIPCC_ERR_INTERNAL: libipcc_err_t = 3; - -/// The requested lookup key was not known to the SP. -pub(crate) const LIBIPCC_ERR_KEY_UNKNOWN: libipcc_err_t = 4; - -/// The value for the requested lookup key was too large for the -/// supplied buffer. -pub(crate) const LIBIPCC_ERR_KEY_BUFTOOSMALL: libipcc_err_t = 5; - -/// An attempt to write to a key failed because the key is read-only. -pub(crate) const LIBIPCC_ERR_KEY_READONLY: libipcc_err_t = 6; - -/// An attempt to write to a key failed because the passed value is too -/// long. -pub(crate) const LIBIPCC_ERR_KEY_VALTOOLONG: libipcc_err_t = 7; - -/// Compression or decompression failed. If appropriate, libipcc_syserr() will -/// return the Z_ error from zlib. -pub(crate) const LIBIPCC_ERR_KEY_ZERR: libipcc_err_t = 8; -pub(crate) type libipcc_err_t = c_uint; - -/// Maxium length of an error message retrieved by libipcc_errmsg(). -pub(crate) const LIBIPCC_ERR_LEN: usize = 1024; - -/// Flags that can be passed to libipcc when looking up a key. Today this is -/// used for looking up a compressed key, however nothing in the public API of -/// this crate takes advantage of this. -pub(crate) type libipcc_key_flag_t = ::std::os::raw::c_uint; - -#[link(name = "ipcc")] -extern "C" { - pub(crate) fn libipcc_init( - lihp: *mut *mut libipcc_handle_t, - libipcc_errp: *mut libipcc_err_t, - syserrp: *mut c_int, - errmsg: *const c_char, - errlen: usize, - ) -> bool; - pub(crate) fn libipcc_fini(lih: *mut libipcc_handle_t); - pub(crate) fn libipcc_err(lih: *mut libipcc_handle_t) -> libipcc_err_t; - pub(crate) fn libipcc_syserr(lih: *mut libipcc_handle_t) -> c_int; - pub(crate) fn libipcc_errmsg(lih: *mut libipcc_handle_t) -> *const c_char; - pub(crate) fn libipcc_keylookup( - lih: *mut libipcc_handle_t, - key: u8, - bufp: *mut *mut u8, - lenp: *mut usize, - flags: libipcc_key_flag_t, - ) -> bool; -} diff --git a/ipcc/src/handle.rs b/ipcc/src/handle.rs deleted file mode 100644 index 91b71a6ce3..0000000000 --- a/ipcc/src/handle.rs +++ /dev/null @@ -1,129 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// Copyright 2023 Oxide Computer Company - -use std::{ - ffi::{c_int, CStr, CString}, - ptr, -}; - -use crate::IpccError; -use crate::{ffi::*, IpccErrorInner}; - -pub struct IpccHandle(*mut libipcc_handle_t); - -impl Drop for IpccHandle { - fn drop(&mut self) { - unsafe { - libipcc_fini(self.0); - } - } -} -fn ipcc_fatal_error>( - context: C, - lerr: libipcc_err_t, - syserr: c_int, - errmsg: CString, -) -> IpccError { - let context = context.into(); - let syserr = if syserr == 0 { - "no system errno".to_string() - } else { - std::io::Error::from_raw_os_error(syserr).to_string() - }; - let inner = IpccErrorInner { - context, - errmsg: errmsg.to_string_lossy().into_owned(), - syserr, - }; - match lerr { - LIBIPCC_ERR_OK => panic!("called fatal on LIBIPCC_ERR_OK"), - LIBIPCC_ERR_NO_MEM => IpccError::NoMem(inner), - LIBIPCC_ERR_INVALID_PARAM => IpccError::InvalidParam(inner), - LIBIPCC_ERR_INTERNAL => IpccError::Internal(inner), - LIBIPCC_ERR_KEY_UNKNOWN => IpccError::KeyUnknown(inner), - LIBIPCC_ERR_KEY_BUFTOOSMALL => IpccError::KeyBufTooSmall(inner), - LIBIPCC_ERR_KEY_READONLY => IpccError::KeyReadonly(inner), - LIBIPCC_ERR_KEY_VALTOOLONG => IpccError::KeyValTooLong(inner), - LIBIPCC_ERR_KEY_ZERR => IpccError::KeyZerr(inner), - _ => IpccError::UnknownErr(inner), - } -} - -impl IpccHandle { - pub fn new() -> Result { - let mut ipcc_handle: *mut libipcc_handle_t = ptr::null_mut(); - // We subtract 1 from the length of the inital vector since CString::new - // will append a nul for us. - // Safety: Unwrapped because we guarantee that the supplied bytes - // contain no 0 bytes up front. - let errmsg = CString::new(vec![1; LIBIPCC_ERR_LEN - 1]).unwrap(); - let errmsg_len = errmsg.as_bytes().len(); - let errmsg_ptr = errmsg.into_raw(); - let mut lerr = LIBIPCC_ERR_OK; - let mut syserr = 0; - if !unsafe { - libipcc_init( - &mut ipcc_handle, - &mut lerr, - &mut syserr, - errmsg_ptr, - errmsg_len, - ) - } { - // Safety: CString::from_raw retakes ownership of a CString - // transferred to C via CString::into_raw. We are calling into_raw() - // above so it is safe to turn this back into it's owned variant. - let errmsg = unsafe { CString::from_raw(errmsg_ptr) }; - return Err(ipcc_fatal_error( - "Could not init libipcc handle", - lerr, - syserr, - errmsg, - )); - } - - Ok(IpccHandle(ipcc_handle)) - } - - fn fatal>(&self, context: C) -> IpccError { - let lerr = unsafe { libipcc_err(self.0) }; - let errmsg = unsafe { libipcc_errmsg(self.0) }; - // Safety: CStr::from_ptr is documented as safe if: - // 1. The pointer contains a valid null terminator at the end of - // the string - // 2. The pointer is valid for reads of bytes up to and including - // the null terminator - // 3. The memory referenced by the return CStr is not mutated for - // the duration of lifetime 'a - // - // (1) is true because this crate initializes space for an error message - // via CString::new which adds a terminator on our behalf. - // (2) should be guaranteed by libipcc itself since it is writing error - // messages into the CString backed buffer that we gave it. - // (3) We aren't currently mutating the memory referenced by the - // CStr, and we are creating an owned copy of the data immediately so - // that it can outlive the lifetime of the libipcc handle if needed. - let errmsg = unsafe { CStr::from_ptr(errmsg) }.to_owned(); - let syserr = unsafe { libipcc_syserr(self.0) }; - ipcc_fatal_error(context, lerr, syserr, errmsg) - } - - pub(crate) fn key_lookup( - &self, - key: u8, - buf: &mut [u8], - ) -> Result { - let mut lenp = buf.len(); - - if !unsafe { - libipcc_keylookup(self.0, key, &mut buf.as_mut_ptr(), &mut lenp, 0) - } { - return Err(self.fatal(format!("lookup of key {key} failed"))); - } - - Ok(lenp) - } -} diff --git a/ipcc/src/handle_stub.rs b/ipcc/src/handle_stub.rs deleted file mode 100644 index bc4b84b7fe..0000000000 --- a/ipcc/src/handle_stub.rs +++ /dev/null @@ -1,25 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// Copyright 2023 Oxide Computer Company - -use crate::IpccError; - -/// This stub and it's implementation are used for non-illumos platforms which -/// lack libipcc. -pub struct IpccHandle; - -impl IpccHandle { - pub fn new() -> Result { - panic!("ipcc unavailable on this platform") - } - - pub(crate) fn key_lookup( - &self, - _key: u8, - _buf: &mut [u8], - ) -> Result { - panic!("ipcc unavailable on this platform") - } -} diff --git a/ipcc/src/lib.rs b/ipcc/src/lib.rs index e997c51230..2693929834 100644 --- a/ipcc/src/lib.rs +++ b/ipcc/src/lib.rs @@ -9,24 +9,13 @@ //! values are variously static, passed from the control plane to the SP //! (through MGS) or set from userland via libipcc. -use cfg_if::cfg_if; +use libipcc::{IpccError, IpccHandle}; use omicron_common::update::ArtifactHash; use serde::Deserialize; use serde::Serialize; use thiserror::Error; use uuid::Uuid; -cfg_if! { - if #[cfg(target_os = "illumos")] { - mod ffi; - mod handle; - use handle::IpccHandle; - } else { - mod handle_stub; - use handle_stub::IpccHandle; - } -} - #[cfg(test)] use proptest::arbitrary::any; #[cfg(test)] @@ -145,36 +134,6 @@ pub enum InstallinatorImageIdError { DeserializationFailed(String), } -#[derive(Error, Debug)] -pub enum IpccError { - #[error("Memory allocation error")] - NoMem(#[source] IpccErrorInner), - #[error("Invalid parameter")] - InvalidParam(#[source] IpccErrorInner), - #[error("Internal error occurred")] - Internal(#[source] IpccErrorInner), - #[error("Requested lookup key was not known to the SP")] - KeyUnknown(#[source] IpccErrorInner), - #[error("Value for the requested lookup key was too large for the supplied buffer")] - KeyBufTooSmall(#[source] IpccErrorInner), - #[error("Attempted to write to read-only key")] - KeyReadonly(#[source] IpccErrorInner), - #[error("Attempted write to key failed because the value is too long")] - KeyValTooLong(#[source] IpccErrorInner), - #[error("Compression or decompression failed")] - KeyZerr(#[source] IpccErrorInner), - #[error("Unknown libipcc error")] - UnknownErr(#[source] IpccErrorInner), -} - -#[derive(Error, Debug)] -#[error("{context}: {errmsg} ({syserr})")] -pub struct IpccErrorInner { - pub context: String, - pub errmsg: String, - pub syserr: String, -} - /// These are the IPCC keys we can look up. /// NB: These keys match the definitions found in libipcc (RFD 316) and should /// match the values in `[ipcc::Key]` one-to-one. diff --git a/live-tests/Cargo.toml b/live-tests/Cargo.toml new file mode 100644 index 0000000000..e0eaf2c338 --- /dev/null +++ b/live-tests/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "omicron-live-tests" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[build-dependencies] +omicron-rpaths.workspace = true + +[dependencies] +# See omicron-rpaths for more about the "pq-sys" dependency. +pq-sys = "*" +omicron-workspace-hack.workspace = true + +[dev-dependencies] +anyhow.workspace = true +assert_matches.workspace = true +dropshot.workspace = true +futures.workspace = true +internal-dns.workspace = true +live-tests-macros.workspace = true +nexus-client.workspace = true +nexus-config.workspace = true +nexus-db-model.workspace = true +nexus-db-queries.workspace = true +nexus-reconfigurator-planning.workspace = true +nexus-reconfigurator-preparation.workspace = true +nexus-sled-agent-shared.workspace = true +nexus-types.workspace = true +omicron-common.workspace = true +omicron-test-utils.workspace = true +reqwest.workspace = true +serde.workspace = true +slog.workspace = true +slog-error-chain.workspace = true +textwrap.workspace = true +tokio.workspace = true +uuid.workspace = true + +[lints] +workspace = true diff --git a/live-tests/README.adoc b/live-tests/README.adoc new file mode 100644 index 0000000000..56f9554bb7 --- /dev/null +++ b/live-tests/README.adoc @@ -0,0 +1,78 @@ += Omicron live tests + +The `omicron-live-tests` package contains automated tests that operate in the context of an already-deployed "real" Oxide system (e.g., `a4x2` or our `london` or `madrid` test environments). This is a home for automated tests for all kinds of Reconfigurator behavior (e.g., add/expunge of all zones, add/expunge sled, upgrades, etc.). It can probably be used for non-Reconfigurator behavior, too. + +This package is not built or tested by default because the tests generally can't work in a dev environment and there's no way to have `cargo` build and check them but not run the tests by default. + +== Why a separate test suite? + +What makes these tests different from the rest of the test suite is that they require connectivity to the underlay network of the deployed system and they make API calls to various components in that system and they assume that this will behave like a real production system. By contrast, the normal tests instead _set up_ a bunch of components using simulated sled agents and localhost networking, which is great for starting from a predictable state and running tests in parallel, but the simulated sled agents and networking make it impossible to exercise quite a lot of Reconfigurator's functionality. + +There are also the `end-to-end-tests`. That environment is more realistic than the main test suite, but not faithful enough for many Reconfigurator tests. + +== Production systems + +There are some safeguards so that these tests won't run on production systems: they refuse to run if they find any Oxide-hardware sleds in the system whose serial numbers don't correspond to known test environments. + +== Usage + +These tests are not currently run automatically (though they are _built_ in CI). + +You can run them yourself. First, deploy Omicron using `a4x2` or one of the hardware test rigs. In your Omicron workspace, run `cargo xtask live-tests` to build an archive and then follow the instructions: + +``` +$ cargo xtask live-tests + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.96s + Running `target/debug/xtask live-tests` +using temporary directory: /dangerzone/omicron_tmp/.tmp0ItZUD +will create archive file: /dangerzone/omicron_tmp/.tmp0ItZUD/live-tests-archive/omicron-live-tests.tar.zst +output tarball: /home/dap/omicron-work/target/live-tests-archive.tgz + +running: /home/dap/.rustup/toolchains/1.80.1-x86_64-unknown-illumos/bin/cargo "nextest" "archive" "--package" "omicron-live-tests" "--archive-file" "/dangerzone/omicron_tmp/.tmp0ItZUD/live-tests-archive/omicron-live-tests.tar.zst" + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.89s +info: experimental features enabled: setup-scripts + Archiving 1 binary, 1 build script output directory, and 1 linked path to /dangerzone/omicron_tmp/.tmp0ItZUD/live-tests-archive/omicron-live-tests.tar.zst + Archived 35 files to /dangerzone/omicron_tmp/.tmp0ItZUD/live-tests-archive/omicron-live-tests.tar.zst in 0.31s +running: bash "-c" "tar cf - Cargo.toml .config/nextest.toml live-tests | tar xf - -C \"/dangerzone/omicron_tmp/.tmp0ItZUD/live-tests-archive\"" +running: tar "cf" "/home/dap/omicron-work/target/live-tests-archive.tgz" "-C" "/dangerzone/omicron_tmp/.tmp0ItZUD" "live-tests-archive" +created: /home/dap/omicron-work/target/live-tests-archive.tgz + +To use this: + +1. Copy the tarball to the switch zone in a deployed Omicron system. + + e.g., scp \ + /home/dap/omicron-work/target/live-tests-archive.tgz \ + root@YOUR_SCRIMLET_GZ_IP:/zone/oxz_switch/root/root + +2. Copy the `cargo-nextest` binary to the same place. + + e.g., scp \ + $(which cargo-nextest) \ + root@YOUR_SCRIMLET_GZ_IP:/zone/oxz_switch/root/root + +3. On that system, unpack the tarball with: + + tar xzf live-tests-archive.tgz + +4. On that system, run tests with: + + TMPDIR=/var/tmp ./cargo-nextest nextest run --profile=live-tests \ + --archive-file live-tests-archive/omicron-live-tests.tar.zst \ + --workspace-remap live-tests-archive +``` + +Follow the instructions, run the tests, and you'll see the usual `nextest`-style output: + +``` +root@oxz_switch:~# TMPDIR=/var/tmp ./cargo-nextest nextest run --archive-file live-tests-archive/omicron-live-tests.tar.zst --workspace-remap live-tests-archive + Extracting 1 binary, 1 build script output directory, and 1 linked path to /var/tmp/nextest-archive-Lqx9VZ + Extracted 35 files to /var/tmp/nextest-archive-Lqx9VZ in 1.01s +info: experimental features enabled: setup-scripts + Starting 1 test across 1 binary (run ID: a5fc9163-9dd5-4b23-b89f-55f8f39ebbbc, nextest profile: default) + SLOW [> 60.000s] omicron-live-tests::test_nexus_add_remove test_nexus_add_remove + PASS [ 61.975s] omicron-live-tests::test_nexus_add_remove test_nexus_add_remove +------------ + Summary [ 61.983s] 1 test run: 1 passed (1 slow), 0 skipped +root@oxz_switch:~# +``` diff --git a/live-tests/build.rs b/live-tests/build.rs new file mode 100644 index 0000000000..1ba9acd41c --- /dev/null +++ b/live-tests/build.rs @@ -0,0 +1,10 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// See omicron-rpaths for documentation. +// NOTE: This file MUST be kept in sync with the other build.rs files in this +// repository. +fn main() { + omicron_rpaths::configure_default_omicron_rpaths(); +} diff --git a/live-tests/macros/Cargo.toml b/live-tests/macros/Cargo.toml new file mode 100644 index 0000000000..81d094d926 --- /dev/null +++ b/live-tests/macros/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "live-tests-macros" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[lib] +proc-macro = true + +[lints] +workspace = true + +[dependencies] +quote.workspace = true +syn = { workspace = true, features = [ "fold", "parsing" ] } +omicron-workspace-hack.workspace = true diff --git a/live-tests/macros/src/lib.rs b/live-tests/macros/src/lib.rs new file mode 100644 index 0000000000..4fdd4029b5 --- /dev/null +++ b/live-tests/macros/src/lib.rs @@ -0,0 +1,86 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Macro to wrap a live test function that automatically creates and cleans up +//! the `LiveTestContext` + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, ItemFn}; + +/// Define a test function that uses `LiveTestContext` +/// +/// This is usable only within the `omicron-live-tests` crate. +/// +/// Similar to `nexus_test`, this macro lets you define a test function that +/// behaves like `tokio::test` except that it accepts an argument of type +/// `&LiveTestContext`. The `LiveTestContext` is cleaned up on _successful_ +/// return of the test function. On failure, debugging information is +/// deliberately left around. +/// +/// Example usage: +/// +/// ```ignore +/// #[live_test] +/// async fn test_my_test_case(lc: &LiveTestContext) { +/// assert!(true); +/// } +/// ``` +/// +/// We use this instead of implementing Drop on LiveTestContext because we want +/// the teardown to only happen when the test doesn't fail (which causes a panic +/// and unwind). +#[proc_macro_attribute] +pub fn live_test(_attrs: TokenStream, input: TokenStream) -> TokenStream { + let input_func = parse_macro_input!(input as ItemFn); + + let mut correct_signature = true; + if input_func.sig.variadic.is_some() + || input_func.sig.inputs.len() != 1 + || input_func.sig.asyncness.is_none() + { + correct_signature = false; + } + + // Verify we're returning an empty tuple + correct_signature &= match input_func.sig.output { + syn::ReturnType::Default => true, + syn::ReturnType::Type(_, ref t) => { + if let syn::Type::Tuple(syn::TypeTuple { elems, .. }) = &**t { + elems.is_empty() + } else { + false + } + } + }; + if !correct_signature { + panic!("func signature must be async fn(&LiveTestContext)"); + } + + let func_ident_string = input_func.sig.ident.to_string(); + let func_ident = input_func.sig.ident.clone(); + let new_block = quote! { + { + #input_func + + let ctx = crate::common::LiveTestContext::new( + #func_ident_string + ).await.expect("setting up LiveTestContext"); + #func_ident(&ctx).await; + ctx.cleanup_successful(); + } + }; + let mut sig = input_func.sig.clone(); + sig.inputs.clear(); + let func = ItemFn { + attrs: input_func.attrs, + vis: input_func.vis, + sig, + block: Box::new(syn::parse2(new_block).unwrap()), + }; + TokenStream::from(quote!( + #[::tokio::test] + #func + )) +} diff --git a/live-tests/tests/common/mod.rs b/live-tests/tests/common/mod.rs new file mode 100644 index 0000000000..28f677f5ed --- /dev/null +++ b/live-tests/tests/common/mod.rs @@ -0,0 +1,249 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +pub mod reconfigurator; + +use anyhow::{anyhow, ensure, Context}; +use dropshot::test_util::LogContext; +use internal_dns::resolver::Resolver; +use internal_dns::ServiceName; +use nexus_config::PostgresConfigWithUrl; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use nexus_types::deployment::SledFilter; +use omicron_common::address::Ipv6Subnet; +use slog::info; +use slog::o; +use std::ffi::OsStr; +use std::net::SocketAddrV6; +use std::path::Component; +use std::sync::Arc; + +/// Contains data and interfaces useful for running tests against an existing +/// deployed control plane +pub struct LiveTestContext { + logctx: LogContext, + opctx: OpContext, + resolver: Resolver, + datastore: Arc, +} + +impl LiveTestContext { + /// Make a new `LiveTestContext` for a test called `test_name`. + pub async fn new( + test_name: &'static str, + ) -> Result { + let logctx = omicron_test_utils::dev::test_setup_log(test_name); + let log = &logctx.log; + let resolver = create_resolver(log)?; + check_execution_environment(&resolver).await?; + let datastore = create_datastore(&log, &resolver).await?; + let opctx = OpContext::for_tests(log.clone(), datastore.clone()); + check_hardware_environment(&opctx, &datastore).await?; + Ok(LiveTestContext { logctx, opctx, resolver, datastore }) + } + + /// Clean up this `LiveTestContext` + /// + /// This mainly removes log files created by the test. We do this in this + /// explicit cleanup function rather than on `Drop` because we want the log + /// files preserved on test failure. + pub fn cleanup_successful(self) { + self.logctx.cleanup_successful(); + } + + /// Returns a logger suitable for use in the test + pub fn log(&self) -> &slog::Logger { + &self.logctx.log + } + + /// Returns an `OpContext` suitable for use in tests + pub fn opctx(&self) -> &OpContext { + &self.opctx + } + + /// Returns a `DataStore` pointing at this deployed system's database + pub fn datastore(&self) -> &DataStore { + &self.datastore + } + + /// Returns a client for a Nexus internal API at the given socket address + pub fn specific_internal_nexus_client( + &self, + sockaddr: SocketAddrV6, + ) -> nexus_client::Client { + let url = format!("http://{}", sockaddr); + let log = self.logctx.log.new(o!("nexus_internal_url" => url.clone())); + nexus_client::Client::new(&url, log) + } + + /// Returns a list of clients for the internal APIs for all Nexus instances + /// found in DNS + pub async fn all_internal_nexus_clients( + &self, + ) -> Result, anyhow::Error> { + Ok(self + .resolver + .lookup_all_socket_v6(ServiceName::Nexus) + .await + .context("looking up Nexus in internal DNS")? + .into_iter() + .map(|s| self.specific_internal_nexus_client(s)) + .collect()) + } +} + +fn create_resolver(log: &slog::Logger) -> Result { + // In principle, we should look at /etc/resolv.conf to find the DNS servers. + // In practice, this usually isn't populated today. See + // oxidecomputer/omicron#2122. + // + // However, the address selected below should work for most existing Omicron + // deployments today. That's because while the base subnet is in principle + // configurable in config-rss.toml, it's very uncommon to change it from the + // default value used here. + let subnet = Ipv6Subnet::new("fd00:1122:3344:0100::".parse().unwrap()); + eprintln!("note: using DNS server for subnet {}", subnet.net()); + internal_dns::resolver::Resolver::new_from_subnet(log.clone(), subnet) + .with_context(|| { + format!("creating DNS resolver for subnet {}", subnet.net()) + }) +} + +/// Creates a DataStore pointing at the CockroachDB cluster that's in DNS +async fn create_datastore( + log: &slog::Logger, + resolver: &Resolver, +) -> Result, anyhow::Error> { + let sockaddrs = resolver + .lookup_all_socket_v6(ServiceName::Cockroach) + .await + .context("resolving CockroachDB")?; + + let url = format!( + "postgresql://root@{}/omicron?sslmode=disable", + sockaddrs + .into_iter() + .map(|a| a.to_string()) + .collect::>() + .join(",") + ) + .parse::() + .context("failed to parse constructed postgres URL")?; + + let db_config = nexus_db_queries::db::Config { url }; + let pool = + Arc::new(nexus_db_queries::db::Pool::new_single_host(log, &db_config)); + DataStore::new_failfast(log, pool) + .await + .context("creating DataStore") + .map(Arc::new) +} + +/// Performs quick checks to determine if the user is running these tests in the +/// wrong place and bails out if so +/// +/// This isn't perfect but seeks to fail fast in obviously bogus environments +/// that someone might accidentally try to run this in. +async fn check_execution_environment( + resolver: &Resolver, +) -> Result<(), anyhow::Error> { + ensure!( + cfg!(target_os = "illumos"), + "live tests can only be run on deployed systems, which run illumos" + ); + + // The only real requirement for these tests is that they're run from a + // place with connectivity to the underlay network of a deployed control + // plane. The easiest way to tell is to look up something in internal DNS. + resolver.lookup_ip(ServiceName::InternalDns).await.map_err(|e| { + let text = format!( + "check_execution_environment(): failed to look up internal DNS \ + in the internal DNS servers.\n\n \ + Are you trying to run this in a development environment? \ + This test can only be run on deployed systems and only from a \ + context with connectivity to the underlay network.\n\n \ + raw error: {}", + slog_error_chain::InlineErrorChain::new(&e) + ); + anyhow!("{}", textwrap::wrap(&text, 80).join("\n")) + })?; + + // Warn the user if the temporary directory is /tmp. This check is + // heuristic. There are other ways they may have specified a tmpfs + // temporary directory and we don't claim to catch all of them. + // + // We could also just go ahead and use /var/tmp, but it's not clear we can + // reliably do that at this point (if Rust or other components have cached + // TMPDIR) and it would be hard to override. + let tmpdir = std::env::temp_dir(); + let mut tmpdir_components = tmpdir.components().take(2); + if let Some(first) = tmpdir_components.next() { + if let Some(next) = tmpdir_components.next() { + if first == Component::RootDir + && next == Component::Normal(OsStr::new("tmp")) + { + eprintln!( + "WARNING: temporary directory appears to be under /tmp, \ + which is generally tmpfs. Consider setting \ + TMPDIR=/var/tmp to avoid runaway tests using too much\ + memory and swap." + ); + } + } + } + + Ok(()) +} + +/// Performs additional checks to determine if we're running in an environment +/// that we believe is safe to run tests +/// +/// These tests may make arbitrary modifications to the system. We don't want +/// to run this in dogfood or other pre-production or production environments. +/// This function uses an allowlist of Oxide serials corresponding to test +/// environments so that it never accidentally runs on a production system. +/// +/// Non-Oxide hardware (e.g., PCs, a4x2, etc.) are always allowed. +async fn check_hardware_environment( + opctx: &OpContext, + datastore: &DataStore, +) -> Result<(), anyhow::Error> { + const ALLOWED_GIMLET_SERIALS: &[&str] = &[ + // test rig: "madrid" + "BRM42220004", + "BRM42220081", + "BRM42220007", + "BRM42220046", + // test rig: "london" + "BRM42220036", + "BRM42220062", + "BRM42220030", + "BRM44220007", + ]; + + // Refuse to operate in an environment that might contain real Oxide + // hardware that's not known to be part of a test rig. This is deliberately + // conservative. + let scary_sleds = datastore + .sled_list_all_batched(opctx, SledFilter::Commissioned) + .await + .context("check_environment: listing commissioned sleds")? + .into_iter() + .filter_map(|s| { + (s.part_number() != "i86pc" + && !ALLOWED_GIMLET_SERIALS.contains(&s.serial_number())) + .then(|| s.serial_number().to_owned()) + }) + .collect::>(); + if scary_sleds.is_empty() { + info!(&opctx.log, "environment verified"); + Ok(()) + } else { + Err(anyhow!( + "refusing to operate in an environment with an unknown system: {}", + scary_sleds.join(", ") + )) + } +} diff --git a/live-tests/tests/common/reconfigurator.rs b/live-tests/tests/common/reconfigurator.rs new file mode 100644 index 0000000000..8f2560bb49 --- /dev/null +++ b/live-tests/tests/common/reconfigurator.rs @@ -0,0 +1,103 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Helpers common to Reconfigurator tests + +use anyhow::{ensure, Context}; +use nexus_client::types::BlueprintTargetSet; +use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; +use nexus_types::deployment::{Blueprint, PlanningInput}; +use slog::{debug, info}; + +/// Modify the system by editing the current target blueprint +/// +/// More precisely, this function: +/// +/// - fetches the current target blueprint +/// - creates a new BlueprintBuilder based on it +/// - invokes the caller's `edit_fn`, which may modify the builder however it +/// likes +/// - generates a new blueprint (thus based on the current target) +/// - uploads the new blueprint +/// - sets the new blueprint as the current target +/// - enables the new blueprint +/// +/// ## Errors +/// +/// This function fails if the current target blueprint is not already enabled. +/// That's because a disabled target blueprint means somebody doesn't want +/// Reconfigurator running or doesn't want it using that blueprint. We don't +/// want the test to inadvertently override that behavior. In a typical use +/// case, a developer enables the initial target blueprint before running these +/// tests and then doesn't need to think about it again for the lifetime of +/// their test environment. +pub async fn blueprint_edit_current_target( + log: &slog::Logger, + planning_input: &PlanningInput, + nexus: &nexus_client::Client, + edit_fn: &dyn Fn(&mut BlueprintBuilder) -> Result<(), anyhow::Error>, +) -> Result<(Blueprint, Blueprint), anyhow::Error> { + // Fetch the current target configuration. + info!(log, "editing current target blueprint"); + let target_blueprint = nexus + .blueprint_target_view() + .await + .context("fetch current target config")? + .into_inner(); + debug!(log, "found current target blueprint"; + "blueprint_id" => %target_blueprint.target_id + ); + ensure!( + target_blueprint.enabled, + "refusing to modify a system with target blueprint disabled" + ); + + // Fetch the actual blueprint. + let blueprint1 = nexus + .blueprint_view(&target_blueprint.target_id) + .await + .context("fetch current target blueprint")? + .into_inner(); + debug!(log, "fetched current target blueprint"; + "blueprint_id" => %target_blueprint.target_id + ); + + // Make a new builder based on that blueprint and use `edit_fn` to edit it. + let mut builder = BlueprintBuilder::new_based_on( + log, + &blueprint1, + &planning_input, + "test-suite", + ) + .context("creating BlueprintBuilder")?; + + edit_fn(&mut builder)?; + + // Assemble the new blueprint, import it, and make it the new target. + let blueprint2 = builder.build(); + info!(log, "assembled new blueprint based on target"; + "current_target_id" => %target_blueprint.target_id, + "new_blueprint_id" => %blueprint2.id, + ); + nexus + .blueprint_import(&blueprint2) + .await + .context("importing new blueprint")?; + debug!(log, "imported new blueprint"; + "blueprint_id" => %blueprint2.id, + ); + nexus + .blueprint_target_set(&BlueprintTargetSet { + enabled: true, + target_id: blueprint2.id, + }) + .await + .expect("setting new target"); + info!(log, "finished editing target blueprint"; + "old_target_id" => %blueprint1.id, + "new_target_id" => %blueprint2.id, + ); + + Ok((blueprint1, blueprint2)) +} diff --git a/live-tests/tests/test_nexus_add_remove.rs b/live-tests/tests/test_nexus_add_remove.rs new file mode 100644 index 0000000000..70e55b704a --- /dev/null +++ b/live-tests/tests/test_nexus_add_remove.rs @@ -0,0 +1,229 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +mod common; + +use anyhow::Context; +use assert_matches::assert_matches; +use common::reconfigurator::blueprint_edit_current_target; +use common::LiveTestContext; +use futures::TryStreamExt; +use live_tests_macros::live_test; +use nexus_client::types::Saga; +use nexus_client::types::SagaState; +use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; +use nexus_reconfigurator_planning::blueprint_builder::EnsureMultiple; +use nexus_reconfigurator_preparation::PlanningInputFromDb; +use nexus_sled_agent_shared::inventory::ZoneKind; +use nexus_types::deployment::SledFilter; +use omicron_common::address::NEXUS_INTERNAL_PORT; +use omicron_test_utils::dev::poll::wait_for_condition; +use omicron_test_utils::dev::poll::CondCheckError; +use slog::{debug, info}; +use std::net::SocketAddrV6; +use std::time::Duration; + +// TODO-coverage This test could check other stuff: +// +// - that after adding: +// - the new Nexus appears in external DNS +// - we can _use_ the new Nexus from the outside +// (e.g., using an `oxide_client` using a custom reqwest resolver that +// points only at that one IP so that we can make sure we're always getting +// that one) +// - that after expungement, it doesn't appear in external DNS any more +// +#[live_test] +async fn test_nexus_add_remove(lc: &LiveTestContext) { + // Test setup + let log = lc.log(); + let opctx = lc.opctx(); + let datastore = lc.datastore(); + let planning_input = PlanningInputFromDb::assemble(&opctx, &datastore) + .await + .expect("planning input"); + let initial_nexus_clients = lc.all_internal_nexus_clients().await.unwrap(); + let nexus = initial_nexus_clients.first().expect("internal Nexus client"); + + // First, deploy a new Nexus zone to an arbitrary sled. + let sled_id = planning_input + .all_sled_ids(SledFilter::Commissioned) + .next() + .expect("any sled id"); + let (blueprint1, blueprint2) = blueprint_edit_current_target( + log, + &planning_input, + &nexus, + &|builder: &mut BlueprintBuilder| { + let nnexus = builder + .sled_num_running_zones_of_kind(sled_id, ZoneKind::Nexus); + let count = builder + .sled_ensure_zone_multiple_nexus(sled_id, nnexus + 1) + .context("adding Nexus zone")?; + assert_matches!( + count, + EnsureMultiple::Changed { added: 1, removed: 0 } + ); + Ok(()) + }, + ) + .await + .expect("editing blueprint to add zone"); + + // Figure out which zone is new and make a new client for it. + let diff = blueprint2.diff_since_blueprint(&blueprint1); + let new_zone = diff + .zones + .added + .values() + .next() + .expect("at least one sled with added zones") + .zones + .first() + .expect("at least one added zone on that sled"); + assert_eq!(new_zone.kind(), ZoneKind::Nexus); + let new_zone_addr = new_zone.underlay_address(); + let new_zone_sockaddr = + SocketAddrV6::new(new_zone_addr, NEXUS_INTERNAL_PORT, 0, 0); + let new_zone_client = lc.specific_internal_nexus_client(new_zone_sockaddr); + + // Wait for the new Nexus zone to show up and be usable. + let initial_sagas_list = wait_for_condition( + || async { + list_sagas(&new_zone_client).await.map_err(|e| { + debug!(log, + "waiting for new Nexus to be available: listing sagas: {e:#}" + ); + CondCheckError::<()>::NotYet + }) + }, + &Duration::from_millis(50), + &Duration::from_secs(60), + ) + .await + .expect("new Nexus to be usable"); + assert!(initial_sagas_list.is_empty()); + info!(log, "new Nexus is online"); + + // Create a demo saga from the new Nexus zone. We'll use this to test that + // when the zone is expunged, its saga gets moved to a different Nexus. + let demo_saga = new_zone_client + .saga_demo_create() + .await + .expect("new Nexus saga demo create"); + let saga_id = demo_saga.saga_id; + let sagas_list = + list_sagas(&new_zone_client).await.expect("new Nexus sagas_list"); + assert_eq!(sagas_list.len(), 1); + assert_eq!(sagas_list[0].id, saga_id); + info!(log, "created demo saga"; "demo_saga" => ?demo_saga); + + // Now expunge the zone we just created. + let _ = blueprint_edit_current_target( + log, + &planning_input, + &nexus, + &|builder: &mut BlueprintBuilder| { + builder + .sled_expunge_zone(sled_id, new_zone.id()) + .context("expunging zone") + }, + ) + .await + .expect("editing blueprint to expunge zone"); + + // At some point, we should be unable to reach this Nexus any more. + wait_for_condition( + || async { + match new_zone_client.saga_list(None, None, None).await { + Err(nexus_client::Error::CommunicationError(error)) => { + info!(log, "expunged Nexus no longer reachable"; + "error" => slog_error_chain::InlineErrorChain::new(&error), + ); + Ok(()) + } + Ok(_) => { + debug!(log, "expunged Nexus is still reachable"); + Err(CondCheckError::<()>::NotYet) + } + Err(error) => { + debug!(log, "expunged Nexus is still reachable"; + "error" => slog_error_chain::InlineErrorChain::new(&error), + ); + Err(CondCheckError::NotYet) + } + } + }, + &Duration::from_millis(50), + &Duration::from_secs(60), + ) + .await + .unwrap(); + + // Wait for some other Nexus instance to pick up the saga. + let nexus_found = wait_for_condition( + || async { + for nexus_client in &initial_nexus_clients { + assert!(nexus_client.baseurl() != new_zone_client.baseurl()); + let Ok(sagas) = list_sagas(&nexus_client).await else { + continue; + }; + + debug!(log, "found sagas (last): {:?}", sagas); + if sagas.into_iter().any(|s| s.id == saga_id) { + return Ok(nexus_client); + } + } + + return Err(CondCheckError::<()>::NotYet); + }, + &Duration::from_millis(50), + &Duration::from_secs(60), + ) + .await + .unwrap(); + + info!(log, "found saga in a different Nexus instance"; + "saga_id" => %saga_id, + "found_nexus" => nexus_found.baseurl(), + ); + assert!(nexus_found.baseurl() != new_zone_client.baseurl()); + + // Now, complete the demo saga on whichever instance is running it now. + // `saga_demo_complete` is not synchronous. It just unblocks the saga. + // We'll need to poll a bit to wait for it to finish. + nexus_found + .saga_demo_complete(&demo_saga.demo_saga_id) + .await + .expect("complete demo saga"); + let found = wait_for_condition( + || async { + let sagas = list_sagas(&nexus_found).await.expect("listing sagas"); + debug!(log, "found sagas (last): {:?}", sagas); + let found = sagas.into_iter().find(|s| s.id == saga_id).unwrap(); + if matches!(found.state, SagaState::Succeeded) { + Ok(found) + } else { + Err(CondCheckError::<()>::NotYet) + } + }, + &Duration::from_millis(50), + &Duration::from_secs(30), + ) + .await + .unwrap(); + + assert_eq!(found.id, saga_id); + assert!(matches!(found.state, SagaState::Succeeded)); +} + +async fn list_sagas( + client: &nexus_client::Client, +) -> Result, anyhow::Error> { + client + .saga_list_stream(None, None) + .try_collect::>() + .await + .context("listing sagas") +} diff --git a/nexus-config/src/postgres_config.rs b/nexus-config/src/postgres_config.rs index 2509ae4fca..0c72d2ba9e 100644 --- a/nexus-config/src/postgres_config.rs +++ b/nexus-config/src/postgres_config.rs @@ -5,6 +5,7 @@ //! Common objects used for configuration use std::fmt; +use std::net::SocketAddr; use std::ops::Deref; use std::str::FromStr; @@ -32,6 +33,29 @@ impl PostgresConfigWithUrl { pub fn url(&self) -> String { self.url_raw.clone() } + + /// Accesses the first ip / port pair within the URL. + /// + /// # Panics + /// + /// This method makes the assumption that the hostname has at least one + /// "host IP / port" pair which can be extracted. If the supplied URL + /// does not have such a pair, this function will panic. + // Yes, panicking in the above scenario sucks. But this type is already + // pretty ubiquitous within Omicron, and integration with the qorb + // connection pooling library requires access to database by SocketAddr. + pub fn address(&self) -> SocketAddr { + let tokio_postgres::config::Host::Tcp(host) = + &self.config.get_hosts()[0] + else { + panic!("Non-TCP hostname"); + }; + let ip: std::net::IpAddr = + host.parse().expect("Failed to parse host as IP address"); + + let port = self.config.get_ports()[0]; + SocketAddr::new(ip, port) + } } impl FromStr for PostgresConfigWithUrl { diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index c7547012ca..cdad883ca7 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -47,6 +47,7 @@ macaddr.workspace = true # integration tests. nexus-client.workspace = true nexus-config.workspace = true +nexus-external-api.workspace = true nexus-internal-api.workspace = true nexus-networking.workspace = true nexus-saga-recovery.workspace = true diff --git a/nexus/db-model/src/dataset.rs b/nexus/db-model/src/dataset.rs index a9dee990b9..f896f11c5b 100644 --- a/nexus/db-model/src/dataset.rs +++ b/nexus/db-model/src/dataset.rs @@ -8,6 +8,7 @@ use crate::ipv6; use crate::schema::{dataset, region}; use chrono::{DateTime, Utc}; use db_macros::Asset; +use omicron_common::api::internal::shared::DatasetKind as ApiDatasetKind; use serde::{Deserialize, Serialize}; use std::net::{Ipv6Addr, SocketAddrV6}; use uuid::Uuid; @@ -41,6 +42,7 @@ pub struct Dataset { pub kind: DatasetKind, pub size_used: Option, + zone_name: Option, } impl Dataset { @@ -48,12 +50,15 @@ impl Dataset { id: Uuid, pool_id: Uuid, addr: Option, - kind: DatasetKind, + api_kind: ApiDatasetKind, ) -> Self { - let size_used = match kind { - DatasetKind::Crucible => Some(0), - _ => None, + let kind = DatasetKind::from(&api_kind); + let (size_used, zone_name) = match api_kind { + ApiDatasetKind::Crucible => (Some(0), None), + ApiDatasetKind::Zone { name } => (None, Some(name)), + _ => (None, None), }; + Self { identity: DatasetIdentity::new(id), time_deleted: None, @@ -63,6 +68,7 @@ impl Dataset { port: addr.map(|addr| addr.port().into()), kind, size_used, + zone_name, } } diff --git a/nexus/db-model/src/dataset_kind.rs b/nexus/db-model/src/dataset_kind.rs index 4a86efaca1..40ec76ded3 100644 --- a/nexus/db-model/src/dataset_kind.rs +++ b/nexus/db-model/src/dataset_kind.rs @@ -23,10 +23,13 @@ impl_enum_type!( ClickhouseServer => b"clickhouse_server" ExternalDns => b"external_dns" InternalDns => b"internal_dns" + ZoneRoot => b"zone_root" + Zone => b"zone" + Debug => b"debug" ); -impl From for DatasetKind { - fn from(k: internal::shared::DatasetKind) -> Self { +impl From<&internal::shared::DatasetKind> for DatasetKind { + fn from(k: &internal::shared::DatasetKind) -> Self { match k { internal::shared::DatasetKind::Crucible => DatasetKind::Crucible, internal::shared::DatasetKind::Cockroach => DatasetKind::Cockroach, @@ -45,6 +48,13 @@ impl From for DatasetKind { internal::shared::DatasetKind::InternalDns => { DatasetKind::InternalDns } + internal::shared::DatasetKind::ZoneRoot => DatasetKind::ZoneRoot, + // Enums in the database do not have associated data, so this drops + // the "name" of the zone and only considers the type. + // + // The zone name, if it exists, is stored in a separate column. + internal::shared::DatasetKind::Zone { .. } => DatasetKind::Zone, + internal::shared::DatasetKind::Debug => DatasetKind::Debug, } } } diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs index 6bef893a5b..b4c60e12ef 100644 --- a/nexus/db-model/src/deployment.rs +++ b/nexus/db-model/src/deployment.rs @@ -6,7 +6,7 @@ //! database use crate::inventory::ZoneType; -use crate::omicron_zone_config::{OmicronZone, OmicronZoneNic}; +use crate::omicron_zone_config::{self, OmicronZoneNic}; use crate::schema::{ blueprint, bp_omicron_physical_disk, bp_omicron_zone, bp_omicron_zone_nic, bp_sled_omicron_physical_disks, bp_sled_omicron_zones, bp_sled_state, @@ -17,21 +17,31 @@ use crate::{ impl_enum_type, ipv6, Generation, MacAddr, Name, SledState, SqlU16, SqlU32, SqlU8, }; +use anyhow::{anyhow, bail, Context, Result}; use chrono::{DateTime, Utc}; use ipnetwork::IpNetwork; -use nexus_types::deployment::BlueprintPhysicalDiskConfig; -use nexus_types::deployment::BlueprintPhysicalDisksConfig; +use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZonesConfig; use nexus_types::deployment::CockroachDbPreserveDowngrade; +use nexus_types::deployment::{ + blueprint_zone_type, BlueprintPhysicalDisksConfig, +}; +use nexus_types::deployment::{BlueprintPhysicalDiskConfig, BlueprintZoneType}; +use nexus_types::deployment::{ + OmicronZoneExternalFloatingAddr, OmicronZoneExternalFloatingIp, + OmicronZoneExternalSnatIp, +}; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::disk::DiskIdentity; -use omicron_uuid_kinds::GenericUuid; +use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use omicron_uuid_kinds::{ExternalIpKind, SledKind, ZpoolKind}; +use omicron_uuid_kinds::{ExternalIpUuid, GenericUuid, OmicronZoneUuid}; +use std::net::{IpAddr, SocketAddrV6}; use uuid::Uuid; /// See [`nexus_types::deployment::Blueprint`]. @@ -256,82 +266,435 @@ impl BpOmicronZone { blueprint_id: Uuid, sled_id: SledUuid, blueprint_zone: &BlueprintZoneConfig, - ) -> Result { + ) -> anyhow::Result { let external_ip_id = blueprint_zone .zone_type .external_networking() - .map(|(ip, _)| ip.id()); - let zone = OmicronZone::new( - sled_id, - blueprint_zone.id.into_untyped_uuid(), - blueprint_zone.underlay_address, - blueprint_zone.filesystem_pool.as_ref().map(|pool| pool.id()), - &blueprint_zone.zone_type.clone().into(), - external_ip_id, - )?; - Ok(Self { + .map(|(ip, _)| ip.id().into()); + + // Create a dummy record to start, then fill in the rest + let mut bp_omicron_zone = BpOmicronZone { + // Fill in the known fields that don't require inspecting + // `blueprint_zone.zone_type` blueprint_id, - sled_id: zone.sled_id.into(), - id: zone.id, - underlay_address: zone.underlay_address, - zone_type: zone.zone_type, - primary_service_ip: zone.primary_service_ip, - primary_service_port: zone.primary_service_port, - second_service_ip: zone.second_service_ip, - second_service_port: zone.second_service_port, - dataset_zpool_name: zone.dataset_zpool_name, - bp_nic_id: zone.nic_id, - dns_gz_address: zone.dns_gz_address, - dns_gz_address_index: zone.dns_gz_address_index, - ntp_ntp_servers: zone.ntp_ntp_servers, - ntp_dns_servers: zone.ntp_dns_servers, - ntp_domain: zone.ntp_domain, - nexus_external_tls: zone.nexus_external_tls, - nexus_external_dns_servers: zone.nexus_external_dns_servers, - snat_ip: zone.snat_ip, - snat_first_port: zone.snat_first_port, - snat_last_port: zone.snat_last_port, - disposition: to_db_bp_zone_disposition(blueprint_zone.disposition), - external_ip_id: zone.external_ip_id.map(From::from), + sled_id: sled_id.into(), + id: blueprint_zone.id.into_untyped_uuid(), + underlay_address: blueprint_zone.underlay_address.into(), + external_ip_id, filesystem_pool: blueprint_zone .filesystem_pool .as_ref() .map(|pool| pool.id().into()), - }) + disposition: to_db_bp_zone_disposition(blueprint_zone.disposition), + zone_type: blueprint_zone.zone_type.kind().into(), + + // Set the remainder of the fields to a default + primary_service_ip: "::1" + .parse::() + .unwrap() + .into(), + primary_service_port: 0.into(), + second_service_ip: None, + second_service_port: None, + dataset_zpool_name: None, + bp_nic_id: None, + dns_gz_address: None, + dns_gz_address_index: None, + ntp_ntp_servers: None, + ntp_dns_servers: None, + ntp_domain: None, + nexus_external_tls: None, + nexus_external_dns_servers: None, + snat_ip: None, + snat_first_port: None, + snat_last_port: None, + }; + + match &blueprint_zone.zone_type { + BlueprintZoneType::BoundaryNtp( + blueprint_zone_type::BoundaryNtp { + address, + ntp_servers, + dns_servers, + domain, + nic, + external_ip, + }, + ) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(address); + + // Set the zone specific fields + let snat_cfg = external_ip.snat_cfg; + let (first_port, last_port) = snat_cfg.port_range_raw(); + bp_omicron_zone.ntp_ntp_servers = Some(ntp_servers.clone()); + bp_omicron_zone.ntp_dns_servers = Some( + dns_servers + .into_iter() + .cloned() + .map(IpNetwork::from) + .collect(), + ); + bp_omicron_zone.ntp_domain.clone_from(domain); + bp_omicron_zone.snat_ip = Some(IpNetwork::from(snat_cfg.ip)); + bp_omicron_zone.snat_first_port = + Some(SqlU16::from(first_port)); + bp_omicron_zone.snat_last_port = Some(SqlU16::from(last_port)); + bp_omicron_zone.bp_nic_id = Some(nic.id); + } + BlueprintZoneType::Clickhouse( + blueprint_zone_type::Clickhouse { address, dataset }, + ) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(address); + bp_omicron_zone.set_zpool_name(dataset); + } + BlueprintZoneType::ClickhouseKeeper( + blueprint_zone_type::ClickhouseKeeper { address, dataset }, + ) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(address); + bp_omicron_zone.set_zpool_name(dataset); + } + BlueprintZoneType::ClickhouseServer( + blueprint_zone_type::ClickhouseServer { address, dataset }, + ) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(address); + bp_omicron_zone.set_zpool_name(dataset); + } + BlueprintZoneType::CockroachDb( + blueprint_zone_type::CockroachDb { address, dataset }, + ) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(address); + bp_omicron_zone.set_zpool_name(dataset); + } + BlueprintZoneType::Crucible(blueprint_zone_type::Crucible { + address, + dataset, + }) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(address); + bp_omicron_zone.set_zpool_name(dataset); + } + BlueprintZoneType::CruciblePantry( + blueprint_zone_type::CruciblePantry { address }, + ) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(address); + } + BlueprintZoneType::ExternalDns( + blueprint_zone_type::ExternalDns { + dataset, + http_address, + dns_address, + nic, + }, + ) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(http_address); + bp_omicron_zone.set_zpool_name(dataset); + + // Set the zone specific fields + bp_omicron_zone.bp_nic_id = Some(nic.id); + bp_omicron_zone.second_service_ip = + Some(IpNetwork::from(dns_address.addr.ip())); + bp_omicron_zone.second_service_port = + Some(SqlU16::from(dns_address.addr.port())); + } + BlueprintZoneType::InternalDns( + blueprint_zone_type::InternalDns { + dataset, + http_address, + dns_address, + gz_address, + gz_address_index, + }, + ) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(http_address); + bp_omicron_zone.set_zpool_name(dataset); + + // Set the zone specific fields + bp_omicron_zone.second_service_ip = + Some(IpNetwork::from(IpAddr::V6(*dns_address.ip()))); + bp_omicron_zone.second_service_port = + Some(SqlU16::from(dns_address.port())); + + bp_omicron_zone.dns_gz_address = + Some(ipv6::Ipv6Addr::from(gz_address)); + bp_omicron_zone.dns_gz_address_index = + Some(SqlU32::from(*gz_address_index)); + } + BlueprintZoneType::InternalNtp( + blueprint_zone_type::InternalNtp { + address, + ntp_servers, + dns_servers, + domain, + }, + ) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(address); + + // Set the zone specific fields + bp_omicron_zone.ntp_ntp_servers = Some(ntp_servers.clone()); + bp_omicron_zone.ntp_dns_servers = Some( + dns_servers.iter().cloned().map(IpNetwork::from).collect(), + ); + bp_omicron_zone.ntp_domain.clone_from(domain); + } + BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { + internal_address, + external_ip, + nic, + external_tls, + external_dns_servers, + }) => { + // Set the common fields + bp_omicron_zone + .set_primary_service_ip_and_port(internal_address); + + // Set the zone specific fields + bp_omicron_zone.bp_nic_id = Some(nic.id); + bp_omicron_zone.second_service_ip = + Some(IpNetwork::from(external_ip.ip)); + bp_omicron_zone.nexus_external_tls = Some(*external_tls); + bp_omicron_zone.nexus_external_dns_servers = Some( + external_dns_servers + .iter() + .cloned() + .map(IpNetwork::from) + .collect(), + ); + } + BlueprintZoneType::Oximeter(blueprint_zone_type::Oximeter { + address, + }) => { + // Set the common fields + bp_omicron_zone.set_primary_service_ip_and_port(address); + } + } + + Ok(bp_omicron_zone) + } + + fn set_primary_service_ip_and_port(&mut self, address: &SocketAddrV6) { + let (primary_service_ip, primary_service_port) = + (ipv6::Ipv6Addr::from(*address.ip()), SqlU16::from(address.port())); + self.primary_service_ip = primary_service_ip; + self.primary_service_port = primary_service_port; + } + + fn set_zpool_name(&mut self, dataset: &OmicronZoneDataset) { + self.dataset_zpool_name = Some(dataset.pool_name.to_string()); + } + /// Convert an external ip from a `BpOmicronZone` to a `BlueprintZoneType` + /// representation. + fn external_ip_to_blueprint_zone_type( + external_ip: Option>, + ) -> anyhow::Result { + external_ip + .map(Into::into) + .ok_or_else(|| anyhow!("expected an external IP ID")) } pub fn into_blueprint_zone_config( self, nic_row: Option, - ) -> Result { - let zone = OmicronZone { - sled_id: self.sled_id.into(), - id: self.id, - underlay_address: self.underlay_address, - filesystem_pool: self.filesystem_pool.map(|id| id.into()), - zone_type: self.zone_type, - primary_service_ip: self.primary_service_ip, - primary_service_port: self.primary_service_port, - second_service_ip: self.second_service_ip, - second_service_port: self.second_service_port, - dataset_zpool_name: self.dataset_zpool_name, - nic_id: self.bp_nic_id, - dns_gz_address: self.dns_gz_address, - dns_gz_address_index: self.dns_gz_address_index, - ntp_ntp_servers: self.ntp_ntp_servers, - ntp_dns_servers: self.ntp_dns_servers, - ntp_domain: self.ntp_domain, - nexus_external_tls: self.nexus_external_tls, - nexus_external_dns_servers: self.nexus_external_dns_servers, - snat_ip: self.snat_ip, - snat_first_port: self.snat_first_port, - snat_last_port: self.snat_last_port, - external_ip_id: self.external_ip_id.map(From::from), + ) -> anyhow::Result { + // Build up a set of common fields for our `BlueprintZoneType`s + // + // Some of these are results that we only evaluate when used, because + // not all zone types use all common fields. + let primary_address = SocketAddrV6::new( + self.primary_service_ip.into(), + *self.primary_service_port, + 0, + 0, + ); + let dataset = + omicron_zone_config::dataset_zpool_name_to_omicron_zone_dataset( + self.dataset_zpool_name, + ); + + // There is a nested result here. If there is a caller error (the outer + // Result) we immediately return. We check the inner result later, but + // only if some code path tries to use `nic` and it's not present. + let nic = omicron_zone_config::nic_row_to_network_interface( + self.id, + self.bp_nic_id, + nic_row.map(Into::into), + )?; + + let external_ip_id = + Self::external_ip_to_blueprint_zone_type(self.external_ip_id); + + let dns_address = + omicron_zone_config::secondary_ip_and_port_to_dns_address( + self.second_service_ip, + self.second_service_port, + ); + + let ntp_dns_servers = + omicron_zone_config::ntp_dns_servers_to_omicron_internal( + self.ntp_dns_servers, + ); + + let ntp_servers = omicron_zone_config::ntp_servers_to_omicron_internal( + self.ntp_ntp_servers, + ); + + let zone_type = match self.zone_type { + ZoneType::BoundaryNtp => { + let snat_cfg = match ( + self.snat_ip, + self.snat_first_port, + self.snat_last_port, + ) { + (Some(ip), Some(first_port), Some(last_port)) => { + nexus_types::inventory::SourceNatConfig::new( + ip.ip(), + *first_port, + *last_port, + ) + .context("bad SNAT config for boundary NTP")? + } + _ => bail!( + "expected non-NULL snat properties, \ + found at least one NULL" + ), + }; + BlueprintZoneType::BoundaryNtp( + blueprint_zone_type::BoundaryNtp { + address: primary_address, + ntp_servers: ntp_servers?, + dns_servers: ntp_dns_servers?, + domain: self.ntp_domain, + nic: nic?, + external_ip: OmicronZoneExternalSnatIp { + id: external_ip_id?, + snat_cfg, + }, + }, + ) + } + ZoneType::Clickhouse => { + BlueprintZoneType::Clickhouse(blueprint_zone_type::Clickhouse { + address: primary_address, + dataset: dataset?, + }) + } + ZoneType::ClickhouseKeeper => BlueprintZoneType::ClickhouseKeeper( + blueprint_zone_type::ClickhouseKeeper { + address: primary_address, + dataset: dataset?, + }, + ), + ZoneType::ClickhouseServer => BlueprintZoneType::ClickhouseServer( + blueprint_zone_type::ClickhouseServer { + address: primary_address, + dataset: dataset?, + }, + ), + + ZoneType::CockroachDb => BlueprintZoneType::CockroachDb( + blueprint_zone_type::CockroachDb { + address: primary_address, + dataset: dataset?, + }, + ), + ZoneType::Crucible => { + BlueprintZoneType::Crucible(blueprint_zone_type::Crucible { + address: primary_address, + dataset: dataset?, + }) + } + ZoneType::CruciblePantry => BlueprintZoneType::CruciblePantry( + blueprint_zone_type::CruciblePantry { + address: primary_address, + }, + ), + ZoneType::ExternalDns => BlueprintZoneType::ExternalDns( + blueprint_zone_type::ExternalDns { + dataset: dataset?, + http_address: primary_address, + dns_address: OmicronZoneExternalFloatingAddr { + id: external_ip_id?, + addr: dns_address?, + }, + nic: nic?, + }, + ), + ZoneType::InternalDns => BlueprintZoneType::InternalDns( + blueprint_zone_type::InternalDns { + dataset: dataset?, + http_address: primary_address, + dns_address: omicron_zone_config::to_internal_dns_address( + dns_address?, + )?, + gz_address: self + .dns_gz_address + .map(Into::into) + .ok_or_else(|| { + anyhow!("expected dns_gz_address, found none") + })?, + gz_address_index: *self.dns_gz_address_index.ok_or_else( + || anyhow!("expected dns_gz_address_index, found none"), + )?, + }, + ), + ZoneType::InternalNtp => BlueprintZoneType::InternalNtp( + blueprint_zone_type::InternalNtp { + address: primary_address, + ntp_servers: ntp_servers?, + dns_servers: ntp_dns_servers?, + domain: self.ntp_domain, + }, + ), + ZoneType::Nexus => { + BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { + internal_address: primary_address, + external_ip: OmicronZoneExternalFloatingIp { + id: external_ip_id?, + ip: self + .second_service_ip + .ok_or_else(|| { + anyhow!("expected second service IP") + })? + .ip(), + }, + nic: nic?, + external_tls: self + .nexus_external_tls + .ok_or_else(|| anyhow!("expected 'external_tls'"))?, + external_dns_servers: self + .nexus_external_dns_servers + .ok_or_else(|| { + anyhow!("expected 'external_dns_servers'") + })? + .into_iter() + .map(|i| i.ip()) + .collect(), + }) + } + ZoneType::Oximeter => { + BlueprintZoneType::Oximeter(blueprint_zone_type::Oximeter { + address: primary_address, + }) + } }; - zone.into_blueprint_zone_config( - self.disposition.into(), - nic_row.map(OmicronZoneNic::from), - ) + + Ok(BlueprintZoneConfig { + disposition: self.disposition.into(), + id: OmicronZoneUuid::from_untyped_uuid(self.id), + underlay_address: self.underlay_address.into(), + filesystem_pool: self + .filesystem_pool + .map(|id| ZpoolName::new_external(id.into())), + zone_type, + }) } } @@ -394,21 +757,6 @@ pub struct BpOmicronZoneNic { slot: SqlU8, } -impl From for OmicronZoneNic { - fn from(value: BpOmicronZoneNic) -> Self { - OmicronZoneNic { - id: value.id, - name: value.name, - ip: value.ip, - mac: value.mac, - subnet: value.subnet, - vni: value.vni, - is_primary: value.is_primary, - slot: value.slot, - } - } -} - impl BpOmicronZoneNic { pub fn new( blueprint_id: Uuid, @@ -440,6 +788,21 @@ impl BpOmicronZoneNic { } } +impl From for OmicronZoneNic { + fn from(value: BpOmicronZoneNic) -> Self { + OmicronZoneNic { + id: value.id, + name: value.name, + ip: value.ip, + mac: value.mac, + subnet: value.subnet, + vni: value.vni, + is_primary: value.is_primary, + slot: value.slot, + } + } +} + mod diesel_util { use crate::{ schema::bp_omicron_zone::disposition, to_db_bp_zone_disposition, diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs index 87986c4f54..71e44b4d82 100644 --- a/nexus/db-model/src/inventory.rs +++ b/nexus/db-model/src/inventory.rs @@ -4,7 +4,7 @@ //! Types for representing the hardware/software inventory in the database -use crate::omicron_zone_config::{OmicronZone, OmicronZoneNic}; +use crate::omicron_zone_config::{self, OmicronZoneNic}; use crate::schema::{ hw_baseboard_id, inv_caboose, inv_collection, inv_collection_error, inv_omicron_zone, inv_omicron_zone_nic, inv_physical_disk, @@ -18,7 +18,7 @@ use crate::{ impl_enum_type, ipv6, ByteCount, Generation, MacAddr, Name, ServiceKind, SqlU16, SqlU32, SqlU8, }; -use anyhow::anyhow; +use anyhow::{anyhow, bail, Context, Result}; use chrono::DateTime; use chrono::Utc; use diesel::backend::Backend; @@ -28,13 +28,15 @@ use diesel::pg::Pg; use diesel::serialize::ToSql; use diesel::{serialize, sql_types}; use ipnetwork::IpNetwork; +use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_sled_agent_shared::inventory::{ - OmicronZoneConfig, OmicronZonesConfig, + OmicronZoneConfig, OmicronZoneType, OmicronZonesConfig, }; use nexus_types::inventory::{ BaseboardId, Caboose, Collection, PowerState, RotPage, RotSlot, }; use omicron_common::api::internal::shared::NetworkInterface; +use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::CollectionKind; use omicron_uuid_kinds::CollectionUuid; use omicron_uuid_kinds::GenericUuid; @@ -42,6 +44,7 @@ use omicron_uuid_kinds::SledKind; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolKind; use omicron_uuid_kinds::ZpoolUuid; +use std::net::{IpAddr, SocketAddrV6}; use uuid::Uuid; // See [`nexus_types::inventory::PowerState`]. @@ -1090,73 +1093,351 @@ impl InvOmicronZone { sled_id: SledUuid, zone: &OmicronZoneConfig, ) -> Result { - // Inventory zones do not know the external IP ID. - let external_ip_id = None; - let zone = OmicronZone::new( - sled_id, - zone.id, - zone.underlay_address, - zone.filesystem_pool.as_ref().map(|pool| pool.id()), - &zone.zone_type, - external_ip_id, - )?; - Ok(Self { + // Create a dummy record to start, then fill in the rest + // according to the zone type + let mut inv_omicron_zone = InvOmicronZone { + // Fill in the known fields that don't require inspecting + // `zone.zone_type` inv_collection_id: inv_collection_id.into(), - sled_id: zone.sled_id.into(), + sled_id: sled_id.into(), id: zone.id, - underlay_address: zone.underlay_address, - zone_type: zone.zone_type, - primary_service_ip: zone.primary_service_ip, - primary_service_port: zone.primary_service_port, - second_service_ip: zone.second_service_ip, - second_service_port: zone.second_service_port, - dataset_zpool_name: zone.dataset_zpool_name, - nic_id: zone.nic_id, - dns_gz_address: zone.dns_gz_address, - dns_gz_address_index: zone.dns_gz_address_index, - ntp_ntp_servers: zone.ntp_ntp_servers, - ntp_dns_servers: zone.ntp_dns_servers, - ntp_domain: zone.ntp_domain, - nexus_external_tls: zone.nexus_external_tls, - nexus_external_dns_servers: zone.nexus_external_dns_servers, - snat_ip: zone.snat_ip, - snat_first_port: zone.snat_first_port, - snat_last_port: zone.snat_last_port, - filesystem_pool: zone.filesystem_pool.map(|id| id.into()), - }) + underlay_address: zone.underlay_address.into(), + filesystem_pool: zone + .filesystem_pool + .as_ref() + .map(|pool| pool.id().into()), + zone_type: zone.zone_type.kind().into(), + + // Set the remainder of the fields to a default + primary_service_ip: "::1" + .parse::() + .unwrap() + .into(), + primary_service_port: 0.into(), + second_service_ip: None, + second_service_port: None, + dataset_zpool_name: None, + nic_id: None, + dns_gz_address: None, + dns_gz_address_index: None, + ntp_ntp_servers: None, + ntp_dns_servers: None, + ntp_domain: None, + nexus_external_tls: None, + nexus_external_dns_servers: None, + snat_ip: None, + snat_first_port: None, + snat_last_port: None, + }; + + match &zone.zone_type { + OmicronZoneType::BoundaryNtp { + address, + ntp_servers, + dns_servers, + domain, + nic, + snat_cfg, + } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(address); + + // Set the zone specific fields + let (first_port, last_port) = snat_cfg.port_range_raw(); + inv_omicron_zone.ntp_ntp_servers = Some(ntp_servers.clone()); + inv_omicron_zone.ntp_dns_servers = Some( + dns_servers + .into_iter() + .cloned() + .map(IpNetwork::from) + .collect(), + ); + inv_omicron_zone.ntp_domain.clone_from(domain); + inv_omicron_zone.snat_ip = Some(IpNetwork::from(snat_cfg.ip)); + inv_omicron_zone.snat_first_port = + Some(SqlU16::from(first_port)); + inv_omicron_zone.snat_last_port = Some(SqlU16::from(last_port)); + inv_omicron_zone.nic_id = Some(nic.id); + } + OmicronZoneType::Clickhouse { address, dataset } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(address); + inv_omicron_zone.set_zpool_name(dataset); + } + OmicronZoneType::ClickhouseKeeper { address, dataset } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(address); + inv_omicron_zone.set_zpool_name(dataset); + } + OmicronZoneType::ClickhouseServer { address, dataset } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(address); + inv_omicron_zone.set_zpool_name(dataset); + } + OmicronZoneType::CockroachDb { address, dataset } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(address); + inv_omicron_zone.set_zpool_name(dataset); + } + OmicronZoneType::Crucible { address, dataset } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(address); + inv_omicron_zone.set_zpool_name(dataset); + } + OmicronZoneType::CruciblePantry { address } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(address); + } + OmicronZoneType::ExternalDns { + dataset, + http_address, + dns_address, + nic, + } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(http_address); + inv_omicron_zone.set_zpool_name(dataset); + + // Set the zone specific fields + inv_omicron_zone.nic_id = Some(nic.id); + inv_omicron_zone.second_service_ip = + Some(IpNetwork::from(dns_address.ip())); + inv_omicron_zone.second_service_port = + Some(SqlU16::from(dns_address.port())); + } + OmicronZoneType::InternalDns { + dataset, + http_address, + dns_address, + gz_address, + gz_address_index, + } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(http_address); + inv_omicron_zone.set_zpool_name(dataset); + + // Set the zone specific fields + inv_omicron_zone.second_service_ip = + Some(IpNetwork::from(IpAddr::V6(*dns_address.ip()))); + inv_omicron_zone.second_service_port = + Some(SqlU16::from(dns_address.port())); + + inv_omicron_zone.dns_gz_address = + Some(ipv6::Ipv6Addr::from(gz_address)); + inv_omicron_zone.dns_gz_address_index = + Some(SqlU32::from(*gz_address_index)); + } + OmicronZoneType::InternalNtp { + address, + ntp_servers, + dns_servers, + domain, + } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(address); + + // Set the zone specific fields + inv_omicron_zone.ntp_ntp_servers = Some(ntp_servers.clone()); + inv_omicron_zone.ntp_dns_servers = Some( + dns_servers.iter().cloned().map(IpNetwork::from).collect(), + ); + inv_omicron_zone.ntp_domain.clone_from(domain); + } + OmicronZoneType::Nexus { + internal_address, + external_ip, + nic, + external_tls, + external_dns_servers, + } => { + // Set the common fields + inv_omicron_zone + .set_primary_service_ip_and_port(internal_address); + + // Set the zone specific fields + inv_omicron_zone.nic_id = Some(nic.id); + inv_omicron_zone.second_service_ip = + Some(IpNetwork::from(*external_ip)); + inv_omicron_zone.nexus_external_tls = Some(*external_tls); + inv_omicron_zone.nexus_external_dns_servers = Some( + external_dns_servers + .iter() + .cloned() + .map(IpNetwork::from) + .collect(), + ); + } + OmicronZoneType::Oximeter { address } => { + // Set the common fields + inv_omicron_zone.set_primary_service_ip_and_port(address); + } + } + + Ok(inv_omicron_zone) + } + + fn set_primary_service_ip_and_port(&mut self, address: &SocketAddrV6) { + let (primary_service_ip, primary_service_port) = + (ipv6::Ipv6Addr::from(*address.ip()), SqlU16::from(address.port())); + self.primary_service_ip = primary_service_ip; + self.primary_service_port = primary_service_port; + } + + fn set_zpool_name(&mut self, dataset: &OmicronZoneDataset) { + self.dataset_zpool_name = Some(dataset.pool_name.to_string()); } pub fn into_omicron_zone_config( self, nic_row: Option, ) -> Result { - let zone = OmicronZone { - sled_id: self.sled_id.into(), - id: self.id, - underlay_address: self.underlay_address, - filesystem_pool: self.filesystem_pool.map(|id| id.into()), - zone_type: self.zone_type, - primary_service_ip: self.primary_service_ip, - primary_service_port: self.primary_service_port, - second_service_ip: self.second_service_ip, - second_service_port: self.second_service_port, - dataset_zpool_name: self.dataset_zpool_name, - nic_id: self.nic_id, - dns_gz_address: self.dns_gz_address, - dns_gz_address_index: self.dns_gz_address_index, - ntp_ntp_servers: self.ntp_ntp_servers, - ntp_dns_servers: self.ntp_dns_servers, - ntp_domain: self.ntp_domain, - nexus_external_tls: self.nexus_external_tls, - nexus_external_dns_servers: self.nexus_external_dns_servers, - snat_ip: self.snat_ip, - snat_first_port: self.snat_first_port, - snat_last_port: self.snat_last_port, - // Inventory zones don't know an external IP ID, and Omicron zone - // configs don't need it. - external_ip_id: None, + // Build up a set of common fields for our `OmicronZoneType`s + // + // Some of these are results that we only evaluate when used, because + // not all zone types use all common fields. + let primary_address = SocketAddrV6::new( + self.primary_service_ip.into(), + *self.primary_service_port, + 0, + 0, + ); + + let dataset = + omicron_zone_config::dataset_zpool_name_to_omicron_zone_dataset( + self.dataset_zpool_name, + ); + + // There is a nested result here. If there is a caller error (the outer + // Result) we immediately return. We check the inner result later, but + // only if some code path tries to use `nic` and it's not present. + let nic = omicron_zone_config::nic_row_to_network_interface( + self.id, + self.nic_id, + nic_row.map(Into::into), + )?; + + let dns_address = + omicron_zone_config::secondary_ip_and_port_to_dns_address( + self.second_service_ip, + self.second_service_port, + ); + + let ntp_dns_servers = + omicron_zone_config::ntp_dns_servers_to_omicron_internal( + self.ntp_dns_servers, + ); + + let ntp_servers = omicron_zone_config::ntp_servers_to_omicron_internal( + self.ntp_ntp_servers, + ); + + let zone_type = match self.zone_type { + ZoneType::BoundaryNtp => { + let snat_cfg = match ( + self.snat_ip, + self.snat_first_port, + self.snat_last_port, + ) { + (Some(ip), Some(first_port), Some(last_port)) => { + nexus_types::inventory::SourceNatConfig::new( + ip.ip(), + *first_port, + *last_port, + ) + .context("bad SNAT config for boundary NTP")? + } + _ => bail!( + "expected non-NULL snat properties, \ + found at least one NULL" + ), + }; + OmicronZoneType::BoundaryNtp { + address: primary_address, + ntp_servers: ntp_servers?, + dns_servers: ntp_dns_servers?, + domain: self.ntp_domain, + nic: nic?, + snat_cfg, + } + } + ZoneType::Clickhouse => OmicronZoneType::Clickhouse { + address: primary_address, + dataset: dataset?, + }, + ZoneType::ClickhouseKeeper => OmicronZoneType::ClickhouseKeeper { + address: primary_address, + dataset: dataset?, + }, + ZoneType::ClickhouseServer => OmicronZoneType::ClickhouseServer { + address: primary_address, + dataset: dataset?, + }, + ZoneType::CockroachDb => OmicronZoneType::CockroachDb { + address: primary_address, + dataset: dataset?, + }, + ZoneType::Crucible => OmicronZoneType::Crucible { + address: primary_address, + dataset: dataset?, + }, + ZoneType::CruciblePantry => { + OmicronZoneType::CruciblePantry { address: primary_address } + } + ZoneType::ExternalDns => OmicronZoneType::ExternalDns { + dataset: dataset?, + http_address: primary_address, + dns_address: dns_address?, + nic: nic?, + }, + ZoneType::InternalDns => OmicronZoneType::InternalDns { + dataset: dataset?, + http_address: primary_address, + dns_address: omicron_zone_config::to_internal_dns_address( + dns_address?, + )?, + gz_address: self.dns_gz_address.map(Into::into).ok_or_else( + || anyhow!("expected dns_gz_address, found none"), + )?, + gz_address_index: *self.dns_gz_address_index.ok_or_else( + || anyhow!("expected dns_gz_address_index, found none"), + )?, + }, + ZoneType::InternalNtp => OmicronZoneType::InternalNtp { + address: primary_address, + ntp_servers: ntp_servers?, + dns_servers: ntp_dns_servers?, + domain: self.ntp_domain, + }, + ZoneType::Nexus => OmicronZoneType::Nexus { + internal_address: primary_address, + external_ip: self + .second_service_ip + .ok_or_else(|| anyhow!("expected second service IP"))? + .ip(), + nic: nic?, + external_tls: self + .nexus_external_tls + .ok_or_else(|| anyhow!("expected 'external_tls'"))?, + external_dns_servers: self + .nexus_external_dns_servers + .ok_or_else(|| anyhow!("expected 'external_dns_servers'"))? + .into_iter() + .map(|i| i.ip()) + .collect(), + }, + ZoneType::Oximeter => { + OmicronZoneType::Oximeter { address: primary_address } + } }; - zone.into_omicron_zone_config(nic_row.map(OmicronZoneNic::from)) + + Ok(OmicronZoneConfig { + id: self.id, + underlay_address: self.underlay_address.into(), + filesystem_pool: self + .filesystem_pool + .map(|id| ZpoolName::new_external(id.into())), + zone_type, + }) } } diff --git a/nexus/db-model/src/omicron_zone_config.rs b/nexus/db-model/src/omicron_zone_config.rs index 23e1ef2dd9..0abc2bb4ec 100644 --- a/nexus/db-model/src/omicron_zone_config.rs +++ b/nexus/db-model/src/omicron_zone_config.rs @@ -2,613 +2,113 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Types for sharing nontrivial conversions between various `OmicronZoneConfig` -//! database serializations and the corresponding Nexus/sled-agent type +//! Helper types and methods for sharing nontrivial conversions between various +//! `OmicronZoneConfig` database serializations and the corresponding Nexus/ +//! sled-agent type //! //! Both inventory and deployment have nearly-identical tables to serialize -//! `OmicronZoneConfigs` that are collected or generated, respectively. We -//! expect those tables to diverge over time (e.g., inventory may start +//! `OmicronZoneConfigs` that are collected or generated, respectively. +//! We expect those tables to diverge over time (e.g., inventory may start //! collecting extra metadata like uptime). This module provides conversion //! helpers for the parts of those tables that are common between the two. -use crate::inventory::ZoneType; -use crate::{ipv6, MacAddr, Name, SqlU16, SqlU32, SqlU8}; +use crate::{MacAddr, Name, SqlU16, SqlU32, SqlU8}; use anyhow::{anyhow, bail, ensure, Context}; use ipnetwork::IpNetwork; -use nexus_sled_agent_shared::inventory::{ - OmicronZoneConfig, OmicronZoneDataset, OmicronZoneType, -}; -use nexus_types::deployment::{ - blueprint_zone_type, BlueprintZoneDisposition, BlueprintZoneType, - OmicronZoneExternalFloatingAddr, OmicronZoneExternalFloatingIp, - OmicronZoneExternalSnatIp, -}; +use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_types::inventory::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; -use omicron_common::zpool_name::ZpoolName; -use omicron_uuid_kinds::{ - ExternalIpUuid, GenericUuid, OmicronZoneUuid, SledUuid, ZpoolUuid, -}; -use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; +use std::net::{IpAddr, SocketAddr, SocketAddrV6}; use uuid::Uuid; -#[derive(Debug)] -pub(crate) struct OmicronZone { - pub(crate) sled_id: SledUuid, - pub(crate) id: Uuid, - pub(crate) underlay_address: ipv6::Ipv6Addr, - pub(crate) filesystem_pool: Option, - pub(crate) zone_type: ZoneType, - pub(crate) primary_service_ip: ipv6::Ipv6Addr, - pub(crate) primary_service_port: SqlU16, - pub(crate) second_service_ip: Option, - pub(crate) second_service_port: Option, - pub(crate) dataset_zpool_name: Option, - pub(crate) nic_id: Option, - pub(crate) dns_gz_address: Option, - pub(crate) dns_gz_address_index: Option, - pub(crate) ntp_ntp_servers: Option>, - pub(crate) ntp_dns_servers: Option>, - pub(crate) ntp_domain: Option, - pub(crate) nexus_external_tls: Option, - pub(crate) nexus_external_dns_servers: Option>, - pub(crate) snat_ip: Option, - pub(crate) snat_first_port: Option, - pub(crate) snat_last_port: Option, - // Only present for BlueprintZoneConfig; always `None` for OmicronZoneConfig - pub(crate) external_ip_id: Option, +/// Convert ntp server config from the DB representation to the +/// omicron internal representation +pub fn ntp_servers_to_omicron_internal( + ntp_ntp_servers: Option>, +) -> anyhow::Result> { + ntp_ntp_servers.ok_or_else(|| anyhow!("expected ntp servers")) } -impl OmicronZone { - pub(crate) fn new( - sled_id: SledUuid, - zone_id: Uuid, - zone_underlay_address: Ipv6Addr, - filesystem_pool: Option, - zone_type: &OmicronZoneType, - external_ip_id: Option, - ) -> anyhow::Result { - let id = zone_id; - let underlay_address = ipv6::Ipv6Addr::from(zone_underlay_address); - let mut nic_id = None; - let mut dns_gz_address = None; - let mut dns_gz_address_index = None; - let mut ntp_ntp_servers = None; - let mut ntp_dns_servers = None; - let mut ntp_ntp_domain = None; - let mut nexus_external_tls = None; - let mut nexus_external_dns_servers = None; - let mut snat_ip = None; - let mut snat_first_port = None; - let mut snat_last_port = None; - let mut second_service_ip = None; - let mut second_service_port = None; - - let (zone_type, primary_service_sockaddr, dataset) = match zone_type { - OmicronZoneType::BoundaryNtp { - address, - ntp_servers, - dns_servers, - domain, - nic, - snat_cfg, - } => { - let (first_port, last_port) = snat_cfg.port_range_raw(); - ntp_ntp_servers = Some(ntp_servers.clone()); - ntp_dns_servers = Some(dns_servers.clone()); - ntp_ntp_domain.clone_from(domain); - snat_ip = Some(IpNetwork::from(snat_cfg.ip)); - snat_first_port = Some(SqlU16::from(first_port)); - snat_last_port = Some(SqlU16::from(last_port)); - nic_id = Some(nic.id); - (ZoneType::BoundaryNtp, address, None) - } - OmicronZoneType::Clickhouse { address, dataset } => { - (ZoneType::Clickhouse, address, Some(dataset)) - } - OmicronZoneType::ClickhouseKeeper { address, dataset } => { - (ZoneType::ClickhouseKeeper, address, Some(dataset)) - } - OmicronZoneType::ClickhouseServer { address, dataset } => { - (ZoneType::ClickhouseServer, address, Some(dataset)) - } - OmicronZoneType::CockroachDb { address, dataset } => { - (ZoneType::CockroachDb, address, Some(dataset)) - } - OmicronZoneType::Crucible { address, dataset } => { - (ZoneType::Crucible, address, Some(dataset)) - } - OmicronZoneType::CruciblePantry { address } => { - (ZoneType::CruciblePantry, address, None) - } - OmicronZoneType::ExternalDns { - dataset, - http_address, - dns_address, - nic, - } => { - nic_id = Some(nic.id); - second_service_ip = Some(dns_address.ip()); - second_service_port = Some(SqlU16::from(dns_address.port())); - (ZoneType::ExternalDns, http_address, Some(dataset)) - } - OmicronZoneType::InternalDns { - dataset, - http_address, - dns_address, - gz_address, - gz_address_index, - } => { - dns_gz_address = Some(ipv6::Ipv6Addr::from(gz_address)); - dns_gz_address_index = Some(SqlU32::from(*gz_address_index)); - second_service_ip = Some(IpAddr::V6(*dns_address.ip())); - second_service_port = Some(SqlU16::from(dns_address.port())); - (ZoneType::InternalDns, http_address, Some(dataset)) - } - OmicronZoneType::InternalNtp { - address, - ntp_servers, - dns_servers, - domain, - } => { - ntp_ntp_servers = Some(ntp_servers.clone()); - ntp_dns_servers = Some(dns_servers.clone()); - ntp_ntp_domain.clone_from(domain); - (ZoneType::InternalNtp, address, None) - } - OmicronZoneType::Nexus { - internal_address, - external_ip, - nic, - external_tls, - external_dns_servers, - } => { - nic_id = Some(nic.id); - nexus_external_tls = Some(*external_tls); - nexus_external_dns_servers = Some(external_dns_servers.clone()); - second_service_ip = Some(*external_ip); - (ZoneType::Nexus, internal_address, None) - } - OmicronZoneType::Oximeter { address } => { - (ZoneType::Oximeter, address, None) - } - }; - - let dataset_zpool_name = dataset.map(|d| d.pool_name.to_string()); - let (primary_service_ip, primary_service_port) = ( - ipv6::Ipv6Addr::from(*primary_service_sockaddr.ip()), - SqlU16::from(primary_service_sockaddr.port()), - ); - - Ok(Self { - sled_id, - id, - underlay_address, - filesystem_pool, - zone_type, - primary_service_ip, - primary_service_port, - second_service_ip: second_service_ip.map(IpNetwork::from), - second_service_port, - dataset_zpool_name, - nic_id, - dns_gz_address, - dns_gz_address_index, - ntp_ntp_servers, - ntp_dns_servers: ntp_dns_servers - .map(|list| list.into_iter().map(IpNetwork::from).collect()), - ntp_domain: ntp_ntp_domain, - nexus_external_tls, - nexus_external_dns_servers: nexus_external_dns_servers - .map(|list| list.into_iter().map(IpNetwork::from).collect()), - snat_ip, - snat_first_port, - snat_last_port, - external_ip_id, - }) - } +/// Convert ntp dns server config from the DB representation to +/// the omicron internal representation. +pub fn ntp_dns_servers_to_omicron_internal( + ntp_dns_servers: Option>, +) -> anyhow::Result> { + ntp_dns_servers + .ok_or_else(|| anyhow!("expected list of DNS servers, found null")) + .map(|list| list.into_iter().map(|ipnetwork| ipnetwork.ip()).collect()) +} - pub(crate) fn into_blueprint_zone_config( - self, - disposition: BlueprintZoneDisposition, - nic_row: Option, - ) -> anyhow::Result { - let common = self.into_zone_config_common(nic_row)?; - let address = common.primary_service_address; - let zone_type = match common.zone_type { - ZoneType::BoundaryNtp => { - let snat_cfg = match ( - common.snat_ip, - common.snat_first_port, - common.snat_last_port, - ) { - (Some(ip), Some(first_port), Some(last_port)) => { - nexus_types::inventory::SourceNatConfig::new( - ip.ip(), - *first_port, - *last_port, - ) - .context("bad SNAT config for boundary NTP")? - } - _ => bail!( - "expected non-NULL snat properties, \ - found at least one NULL" - ), - }; - BlueprintZoneType::BoundaryNtp( - blueprint_zone_type::BoundaryNtp { - address, - dns_servers: common.ntp_dns_servers?, - domain: common.ntp_domain, - nic: common.nic?, - ntp_servers: common.ntp_ntp_servers?, - external_ip: OmicronZoneExternalSnatIp { - id: common.external_ip_id?, - snat_cfg, - }, - }, - ) - } - ZoneType::Clickhouse => { - BlueprintZoneType::Clickhouse(blueprint_zone_type::Clickhouse { - address, - dataset: common.dataset?, - }) - } - ZoneType::ClickhouseKeeper => BlueprintZoneType::ClickhouseKeeper( - blueprint_zone_type::ClickhouseKeeper { - address, - dataset: common.dataset?, - }, - ), - ZoneType::ClickhouseServer => BlueprintZoneType::ClickhouseServer( - blueprint_zone_type::ClickhouseServer { - address, - dataset: common.dataset?, - }, - ), - ZoneType::CockroachDb => BlueprintZoneType::CockroachDb( - blueprint_zone_type::CockroachDb { - address, - dataset: common.dataset?, - }, - ), - ZoneType::Crucible => { - BlueprintZoneType::Crucible(blueprint_zone_type::Crucible { - address, - dataset: common.dataset?, - }) - } - ZoneType::CruciblePantry => BlueprintZoneType::CruciblePantry( - blueprint_zone_type::CruciblePantry { address }, - ), - ZoneType::ExternalDns => BlueprintZoneType::ExternalDns( - blueprint_zone_type::ExternalDns { - dataset: common.dataset?, - dns_address: OmicronZoneExternalFloatingAddr { - id: common.external_ip_id?, - addr: common.dns_address?, - }, - http_address: address, - nic: common.nic?, - }, - ), - ZoneType::InternalDns => BlueprintZoneType::InternalDns( - blueprint_zone_type::InternalDns { - dataset: common.dataset?, - dns_address: to_internal_dns_address(common.dns_address?)?, - http_address: address, - gz_address: *common.dns_gz_address.ok_or_else(|| { - anyhow!("expected dns_gz_address, found none") - })?, - gz_address_index: *common.dns_gz_address_index.ok_or_else( - || anyhow!("expected dns_gz_address_index, found none"), - )?, - }, - ), - ZoneType::InternalNtp => BlueprintZoneType::InternalNtp( - blueprint_zone_type::InternalNtp { - address, - dns_servers: common.ntp_dns_servers?, - domain: common.ntp_domain, - ntp_servers: common.ntp_ntp_servers?, - }, - ), - ZoneType::Nexus => { - BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { - internal_address: address, - nic: common.nic?, - external_tls: common - .nexus_external_tls - .ok_or_else(|| anyhow!("expected 'external_tls'"))?, - external_ip: OmicronZoneExternalFloatingIp { - id: common.external_ip_id?, - ip: common - .second_service_ip - .ok_or_else(|| { - anyhow!("expected second service IP") - })? - .ip(), - }, - external_dns_servers: common - .nexus_external_dns_servers - .ok_or_else(|| { - anyhow!("expected 'external_dns_servers'") - })? - .into_iter() - .map(|i| i.ip()) - .collect(), - }) - } - ZoneType::Oximeter => { - BlueprintZoneType::Oximeter(blueprint_zone_type::Oximeter { - address, - }) - } - }; - Ok(nexus_types::deployment::BlueprintZoneConfig { - disposition, - id: OmicronZoneUuid::from_untyped_uuid(common.id), - underlay_address: std::net::Ipv6Addr::from(common.underlay_address), - filesystem_pool: common - .filesystem_pool - .map(|id| ZpoolName::new_external(id)), - zone_type, - }) +/// Assemble a value that we can use to extract the NIC _if necessary_ +/// and report an error if it was needed but not found. +/// +/// Any error here should be impossible. By the time we get here, the +/// caller should have provided `nic_row` iff there's a corresponding +/// `nic_id` in this row, and the ids should match up. And whoever +/// created this row ought to have provided a nic_id iff this type of +/// zone needs a NIC. This last issue is not under our control, though, +/// so we definitely want to handle that as an operational error. The +/// others could arguably be programmer errors (i.e., we could `assert`), +/// but it seems excessive to crash here. +/// +/// The outer result represents a programmer error and should be unwrapped +/// immediately. The inner result represents an operational error and should +/// only be unwrapped when the nic is used. +pub fn nic_row_to_network_interface( + zone_id: Uuid, + nic_id: Option, + nic_row: Option, +) -> anyhow::Result> { + match (nic_id, nic_row) { + (Some(expected_id), Some(nic_row)) => { + ensure!(expected_id == nic_row.id, "caller provided wrong NIC"); + Ok(nic_row.into_network_interface_for_zone(zone_id)) + } + (None, None) => Ok(Err(anyhow!( + "expected zone to have an associated NIC, but it doesn't" + ))), + (Some(_), None) => bail!("caller provided no NIC"), + (None, Some(_)) => bail!("caller unexpectedly provided a NIC"), } +} - pub(crate) fn into_omicron_zone_config( - self, - nic_row: Option, - ) -> anyhow::Result { - let common = self.into_zone_config_common(nic_row)?; - let address = common.primary_service_address; - - let zone_type = match common.zone_type { - ZoneType::BoundaryNtp => { - let snat_cfg = match ( - common.snat_ip, - common.snat_first_port, - common.snat_last_port, - ) { - (Some(ip), Some(first_port), Some(last_port)) => { - nexus_types::inventory::SourceNatConfig::new( - ip.ip(), - *first_port, - *last_port, - ) - .context("bad SNAT config for boundary NTP")? - } - _ => bail!( - "expected non-NULL snat properties, \ - found at least one NULL" - ), - }; - OmicronZoneType::BoundaryNtp { - address, - dns_servers: common.ntp_dns_servers?, - domain: common.ntp_domain, - nic: common.nic?, - ntp_servers: common.ntp_ntp_servers?, - snat_cfg, - } - } - ZoneType::Clickhouse => OmicronZoneType::Clickhouse { - address, - dataset: common.dataset?, - }, - ZoneType::ClickhouseKeeper => OmicronZoneType::ClickhouseKeeper { - address, - dataset: common.dataset?, - }, - ZoneType::ClickhouseServer => OmicronZoneType::ClickhouseServer { - address, - dataset: common.dataset?, - }, - ZoneType::CockroachDb => OmicronZoneType::CockroachDb { - address, - dataset: common.dataset?, - }, - ZoneType::Crucible => { - OmicronZoneType::Crucible { address, dataset: common.dataset? } - } - ZoneType::CruciblePantry => { - OmicronZoneType::CruciblePantry { address } - } - ZoneType::ExternalDns => OmicronZoneType::ExternalDns { - dataset: common.dataset?, - dns_address: common.dns_address?, - http_address: address, - nic: common.nic?, - }, - ZoneType::InternalDns => OmicronZoneType::InternalDns { - dataset: common.dataset?, - dns_address: to_internal_dns_address(common.dns_address?)?, - http_address: address, - gz_address: *common.dns_gz_address.ok_or_else(|| { - anyhow!("expected dns_gz_address, found none") +/// Convert a dataset from a DB representation to a an Omicron internal +/// representation +pub fn dataset_zpool_name_to_omicron_zone_dataset( + dataset_zpool_name: Option, +) -> anyhow::Result { + dataset_zpool_name + .map(|zpool_name| -> Result<_, anyhow::Error> { + Ok(OmicronZoneDataset { + pool_name: zpool_name.parse().map_err(|e| { + anyhow!("parsing zpool name {:?}: {}", zpool_name, e) })?, - gz_address_index: *common.dns_gz_address_index.ok_or_else( - || anyhow!("expected dns_gz_address_index, found none"), - )?, - }, - ZoneType::InternalNtp => OmicronZoneType::InternalNtp { - address, - dns_servers: common.ntp_dns_servers?, - domain: common.ntp_domain, - ntp_servers: common.ntp_ntp_servers?, - }, - ZoneType::Nexus => OmicronZoneType::Nexus { - internal_address: address, - nic: common.nic?, - external_tls: common - .nexus_external_tls - .ok_or_else(|| anyhow!("expected 'external_tls'"))?, - external_ip: common - .second_service_ip - .ok_or_else(|| anyhow!("expected second service IP"))? - .ip(), - external_dns_servers: common - .nexus_external_dns_servers - .ok_or_else(|| anyhow!("expected 'external_dns_servers'"))? - .into_iter() - .map(|i| i.ip()) - .collect(), - }, - ZoneType::Oximeter => OmicronZoneType::Oximeter { address }, - }; - Ok(OmicronZoneConfig { - id: common.id, - underlay_address: std::net::Ipv6Addr::from(common.underlay_address), - filesystem_pool: common - .filesystem_pool - .map(|id| ZpoolName::new_external(id)), - zone_type, - }) - } - - fn into_zone_config_common( - self, - nic_row: Option, - ) -> anyhow::Result { - let primary_service_address = SocketAddrV6::new( - std::net::Ipv6Addr::from(self.primary_service_ip), - *self.primary_service_port, - 0, - 0, - ); - - // Assemble a value that we can use to extract the NIC _if necessary_ - // and report an error if it was needed but not found. - // - // Any error here should be impossible. By the time we get here, the - // caller should have provided `nic_row` iff there's a corresponding - // `nic_id` in this row, and the ids should match up. And whoever - // created this row ought to have provided a nic_id iff this type of - // zone needs a NIC. This last issue is not under our control, though, - // so we definitely want to handle that as an operational error. The - // others could arguably be programmer errors (i.e., we could `assert`), - // but it seems excessive to crash here. - // - // Note that we immediately return for any of the caller errors here. - // For the other error, we will return only later, if some code path - // below tries to use `nic` when it's not present. - let nic = match (self.nic_id, nic_row) { - (Some(expected_id), Some(nic_row)) => { - ensure!(expected_id == nic_row.id, "caller provided wrong NIC"); - Ok(nic_row.into_network_interface_for_zone(self.id)?) - } - // We don't expect and don't have a NIC. This is reasonable, so we - // don't `bail!` like we do in the next two cases, but we also - // _don't have a NIC_. Put an error into `nic`, and then if we land - // in a zone below that expects one, we'll fail then. - (None, None) => Err(anyhow!( - "expected zone to have an associated NIC, but it doesn't" - )), - (Some(_), None) => bail!("caller provided no NIC"), - (None, Some(_)) => bail!("caller unexpectedly provided a NIC"), - }; - - // Similarly, assemble a value that we can use to extract the dataset, - // if necessary. We only return this error if code below tries to use - // this value. - let dataset = self - .dataset_zpool_name - .map(|zpool_name| -> Result<_, anyhow::Error> { - Ok(OmicronZoneDataset { - pool_name: zpool_name.parse().map_err(|e| { - anyhow!("parsing zpool name {:?}: {}", zpool_name, e) - })?, - }) }) - .transpose()? - .ok_or_else(|| anyhow!("expected dataset zpool name, found none")); - - // Do the same for the DNS server address. - let dns_address = - match (self.second_service_ip, self.second_service_port) { - (Some(dns_ip), Some(dns_port)) => { - Ok(std::net::SocketAddr::new(dns_ip.ip(), *dns_port)) - } - _ => Err(anyhow!( - "expected second service IP and port, \ - found one missing" - )), - }; - - // Do the same for NTP zone properties. - let ntp_dns_servers = self - .ntp_dns_servers - .ok_or_else(|| anyhow!("expected list of DNS servers, found null")) - .map(|list| { - list.into_iter().map(|ipnetwork| ipnetwork.ip()).collect() - }); - let ntp_ntp_servers = - self.ntp_ntp_servers.ok_or_else(|| anyhow!("expected ntp_servers")); - - // Do the same for the external IP ID. - let external_ip_id = - self.external_ip_id.context("expected an external IP ID"); - - Ok(ZoneConfigCommon { - id: self.id, - underlay_address: self.underlay_address, - filesystem_pool: self.filesystem_pool, - zone_type: self.zone_type, - primary_service_address, - snat_ip: self.snat_ip, - snat_first_port: self.snat_first_port, - snat_last_port: self.snat_last_port, - ntp_domain: self.ntp_domain, - dns_gz_address: self.dns_gz_address, - dns_gz_address_index: self.dns_gz_address_index, - nexus_external_tls: self.nexus_external_tls, - nexus_external_dns_servers: self.nexus_external_dns_servers, - second_service_ip: self.second_service_ip, - nic, - dataset, - dns_address, - ntp_dns_servers, - ntp_ntp_servers, - external_ip_id, }) - } + .transpose()? + .ok_or_else(|| anyhow!("expected dataset zpool name, found none")) } -struct ZoneConfigCommon { - id: Uuid, - underlay_address: ipv6::Ipv6Addr, - filesystem_pool: Option, - zone_type: ZoneType, - primary_service_address: SocketAddrV6, - snat_ip: Option, - snat_first_port: Option, - snat_last_port: Option, - ntp_domain: Option, - dns_gz_address: Option, - dns_gz_address_index: Option, - nexus_external_tls: Option, - nexus_external_dns_servers: Option>, +/// Convert the secondary ip and port to a dns address +pub fn secondary_ip_and_port_to_dns_address( second_service_ip: Option, - // These properties may or may not be needed, depending on the zone type. We - // store results here that can be unpacked once we determine our zone type. - nic: anyhow::Result, - dataset: anyhow::Result, - // Note that external DNS is SocketAddr (also supports v4) while internal - // DNS is always v6. - dns_address: anyhow::Result, - ntp_dns_servers: anyhow::Result>, - ntp_ntp_servers: anyhow::Result>, - external_ip_id: anyhow::Result, + second_service_port: Option, +) -> anyhow::Result { + match (second_service_ip, second_service_port) { + (Some(dns_ip), Some(dns_port)) => { + Ok(std::net::SocketAddr::new(dns_ip.ip(), *dns_port)) + } + _ => Err(anyhow!( + "expected second service IP and port, found one missing" + )), + } } -// Ideally this would be a method on `ZoneConfigCommon`, but that's more -// annoying to deal with because often, at the time this function is called, -// part of `ZoneConfigCommon` has already been moved out. -fn to_internal_dns_address( - external_address: SocketAddr, +/// Extract a SocketAddrV6 from a SocketAddr for a given dns address +pub fn to_internal_dns_address( + address: SocketAddr, ) -> anyhow::Result { - match external_address { + match address { SocketAddr::V4(address) => { bail!( "expected internal DNS address to be v6, found v4: {:?}", diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index f01f33c39d..5d9b3da78f 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1023,6 +1023,7 @@ table! { kind -> crate::DatasetKindEnum, size_used -> Nullable, + zone_name -> Nullable, } } diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index eaed2990c5..2438f37fba 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(92, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(93, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(93, "dataset-kinds-zone-and-debug"), KnownVersion::new(92, "lldp-link-config-nullable"), KnownVersion::new(91, "add-management-gateway-producer-kind"), KnownVersion::new(90, "lookup-bgp-config-by-asn"), diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index 5192528944..c6c5caab6a 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -14,7 +14,6 @@ omicron-rpaths.workspace = true anyhow.workspace = true async-bb8-diesel.workspace = true async-trait.workspace = true -bb8.workspace = true camino.workspace = true chrono.workspace = true const_format.workspace = true @@ -22,6 +21,7 @@ diesel.workspace = true diesel-dtrace.workspace = true dropshot.workspace = true futures.workspace = true +internal-dns.workspace = true ipnetwork.workspace = true macaddr.workspace = true once_cell.workspace = true @@ -29,6 +29,7 @@ oxnet.workspace = true paste.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. pq-sys = "*" +qorb = { workspace = true, features = [ "qtop" ] } rand.workspace = true ref-cast.workspace = true schemars.workspace = true @@ -45,8 +46,9 @@ strum.workspace = true swrite.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["full"] } -uuid.workspace = true +url.workspace = true usdt.workspace = true +uuid.workspace = true db-macros.workspace = true nexus-auth.workspace = true diff --git a/nexus/db-queries/src/db/collection_attach.rs b/nexus/db-queries/src/db/collection_attach.rs index 95e6afeb4b..c009d60483 100644 --- a/nexus/db-queries/src/db/collection_attach.rs +++ b/nexus/db-queries/src/db/collection_attach.rs @@ -578,9 +578,7 @@ where mod test { use super::*; use crate::db::{self, identity::Resource as IdentityResource}; - use async_bb8_diesel::{ - AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use chrono::Utc; use db_macros::Resource; use diesel::expression_methods::ExpressionMethods; @@ -617,8 +615,8 @@ mod test { async fn setup_db( pool: &crate::db::Pool, - ) -> bb8::PooledConnection> { - let connection = pool.pool().get().await.unwrap(); + ) -> crate::db::datastore::DataStoreConnection { + let connection = pool.claim().await.unwrap(); (*connection) .batch_execute_async( "CREATE SCHEMA IF NOT EXISTS test_schema; \ @@ -873,7 +871,7 @@ mod test { dev::test_setup_log("test_attach_missing_collection_fails"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -902,7 +900,7 @@ mod test { let logctx = dev::test_setup_log("test_attach_missing_resource_fails"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -939,7 +937,7 @@ mod test { let logctx = dev::test_setup_log("test_attach_once"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -987,7 +985,7 @@ mod test { let logctx = dev::test_setup_log("test_attach_once_synchronous"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -1036,7 +1034,7 @@ mod test { let logctx = dev::test_setup_log("test_attach_multiple_times"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -1092,7 +1090,7 @@ mod test { let logctx = dev::test_setup_log("test_attach_beyond_capacity_fails"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -1156,7 +1154,7 @@ mod test { let logctx = dev::test_setup_log("test_attach_while_already_attached"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -1263,7 +1261,7 @@ mod test { let logctx = dev::test_setup_log("test_attach_once"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -1318,7 +1316,7 @@ mod test { let logctx = dev::test_setup_log("test_attach_deleted_resource_fails"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -1363,7 +1361,7 @@ mod test { let logctx = dev::test_setup_log("test_attach_without_update_filter"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; diff --git a/nexus/db-queries/src/db/collection_detach.rs b/nexus/db-queries/src/db/collection_detach.rs index 03e09d41ca..bc547d5127 100644 --- a/nexus/db-queries/src/db/collection_detach.rs +++ b/nexus/db-queries/src/db/collection_detach.rs @@ -482,9 +482,7 @@ mod test { use super::*; use crate::db::collection_attach::DatastoreAttachTarget; use crate::db::{self, identity::Resource as IdentityResource}; - use async_bb8_diesel::{ - AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use chrono::Utc; use db_macros::Resource; use diesel::expression_methods::ExpressionMethods; @@ -521,8 +519,8 @@ mod test { async fn setup_db( pool: &crate::db::Pool, - ) -> bb8::PooledConnection> { - let connection = pool.pool().get().await.unwrap(); + ) -> crate::db::datastore::DataStoreConnection { + let connection = pool.claim().await.unwrap(); (*connection) .batch_execute_async( "CREATE SCHEMA IF NOT EXISTS test_schema; \ @@ -786,7 +784,7 @@ mod test { dev::test_setup_log("test_detach_missing_collection_fails"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -814,7 +812,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_missing_resource_fails"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -850,7 +848,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_once"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -890,7 +888,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_while_already_detached"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -954,7 +952,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_deleted_resource_fails"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -998,7 +996,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_without_update_filter"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; diff --git a/nexus/db-queries/src/db/collection_detach_many.rs b/nexus/db-queries/src/db/collection_detach_many.rs index 986cfb70b7..36755599d4 100644 --- a/nexus/db-queries/src/db/collection_detach_many.rs +++ b/nexus/db-queries/src/db/collection_detach_many.rs @@ -480,9 +480,7 @@ mod test { use super::*; use crate::db::collection_attach::DatastoreAttachTarget; use crate::db::{self, identity::Resource as IdentityResource}; - use async_bb8_diesel::{ - AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use chrono::Utc; use db_macros::Resource; use diesel::expression_methods::ExpressionMethods; @@ -519,8 +517,8 @@ mod test { async fn setup_db( pool: &crate::db::Pool, - ) -> bb8::PooledConnection> { - let connection = pool.pool().get().await.unwrap(); + ) -> crate::db::datastore::DataStoreConnection { + let connection = pool.claim().await.unwrap(); (*connection) .batch_execute_async( "CREATE SCHEMA IF NOT EXISTS test_schema; \ @@ -778,7 +776,7 @@ mod test { dev::test_setup_log("test_detach_missing_collection_fails"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -808,7 +806,7 @@ mod test { dev::test_setup_log("test_detach_missing_resource_succeeds"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -849,7 +847,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_once"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -892,7 +890,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_once_synchronous"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -937,7 +935,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_while_already_detached"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -993,7 +991,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_filter_collection"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -1044,7 +1042,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_deleted_resource"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -1102,7 +1100,7 @@ mod test { let logctx = dev::test_setup_log("test_detach_many"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; diff --git a/nexus/db-queries/src/db/collection_insert.rs b/nexus/db-queries/src/db/collection_insert.rs index 69906e6498..3aaea6aeb1 100644 --- a/nexus/db-queries/src/db/collection_insert.rs +++ b/nexus/db-queries/src/db/collection_insert.rs @@ -406,9 +406,7 @@ where mod test { use super::*; use crate::db::{self, identity::Resource as IdentityResource}; - use async_bb8_diesel::{ - AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use chrono::{DateTime, Utc}; use db_macros::Resource; use diesel::expression_methods::ExpressionMethods; @@ -443,8 +441,8 @@ mod test { async fn setup_db( pool: &crate::db::Pool, - ) -> bb8::PooledConnection> { - let connection = pool.pool().get().await.unwrap(); + ) -> crate::db::datastore::DataStoreConnection { + let connection = pool.claim().await.unwrap(); (*connection) .batch_execute_async( "CREATE SCHEMA IF NOT EXISTS test_schema; \ @@ -560,7 +558,7 @@ mod test { let logctx = dev::test_setup_log("test_collection_not_present"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; @@ -590,7 +588,7 @@ mod test { let logctx = dev::test_setup_log("test_collection_present"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let conn = setup_db(&pool).await; diff --git a/nexus/db-queries/src/db/datastore/dataset.rs b/nexus/db-queries/src/db/datastore/dataset.rs index a08e346fe8..0fe1c7912e 100644 --- a/nexus/db-queries/src/db/datastore/dataset.rs +++ b/nexus/db-queries/src/db/datastore/dataset.rs @@ -241,6 +241,7 @@ mod test { use nexus_db_model::SledSystemHardware; use nexus_db_model::SledUpdate; use nexus_test_utils::db::test_setup_database; + use omicron_common::api::internal::shared::DatasetKind as ApiDatasetKind; use omicron_test_utils::dev; #[tokio::test] @@ -291,7 +292,7 @@ mod test { Uuid::new_v4(), zpool_id, Some("[::1]:0".parse().unwrap()), - DatasetKind::Crucible, + ApiDatasetKind::Crucible, )) .await .expect("failed to insert dataset") @@ -324,7 +325,7 @@ mod test { dataset1.id(), zpool_id, Some("[::1]:12345".parse().unwrap()), - DatasetKind::Cockroach, + ApiDatasetKind::Cockroach, )) .await .expect("failed to do-nothing insert dataset"); @@ -340,7 +341,7 @@ mod test { Uuid::new_v4(), zpool_id, Some("[::1]:0".parse().unwrap()), - DatasetKind::Cockroach, + ApiDatasetKind::Cockroach, )) .await .expect("failed to upsert dataset"); @@ -372,7 +373,7 @@ mod test { dataset1.id(), zpool_id, Some("[::1]:12345".parse().unwrap()), - DatasetKind::Cockroach, + ApiDatasetKind::Cockroach, )) .await .expect("failed to do-nothing insert dataset"); diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs index 4169cc06bd..b997bf384f 100644 --- a/nexus/db-queries/src/db/datastore/db_metadata.rs +++ b/nexus/db-queries/src/db/datastore/db_metadata.rs @@ -511,7 +511,7 @@ mod test { let mut crdb = test_db::test_setup_database(&logctx.log).await; let cfg = db::Config { url: crdb.pg_config().clone() }; - let pool = Arc::new(db::Pool::new(&logctx.log, &cfg)); + let pool = Arc::new(db::Pool::new_single_host(&logctx.log, &cfg)); let datastore = Arc::new(DataStore::new(&logctx.log, pool, None).await.unwrap()); @@ -559,8 +559,8 @@ mod test { let mut crdb = test_db::test_setup_database(&logctx.log).await; let cfg = db::Config { url: crdb.pg_config().clone() }; - let pool = Arc::new(db::Pool::new(&logctx.log, &cfg)); - let conn = pool.pool().get().await.unwrap(); + let pool = Arc::new(db::Pool::new_single_host(&logctx.log, &cfg)); + let conn = pool.claim().await.unwrap(); // Mimic the layout of "schema/crdb". let config_dir = Utf8TempDir::new().unwrap(); @@ -671,8 +671,8 @@ mod test { let mut crdb = test_db::test_setup_database(&logctx.log).await; let cfg = db::Config { url: crdb.pg_config().clone() }; - let pool = Arc::new(db::Pool::new(&logctx.log, &cfg)); - let conn = pool.pool().get().await.unwrap(); + let pool = Arc::new(db::Pool::new_single_host(&logctx.log, &cfg)); + let conn = pool.claim().await.unwrap(); // Mimic the layout of "schema/crdb". let config_dir = Utf8TempDir::new().unwrap(); diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 1774a25c48..8888f2caaa 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -2164,7 +2164,7 @@ mod test { } impl CollectionCounts { - async fn new(conn: &DataStoreConnection<'_>) -> anyhow::Result { + async fn new(conn: &DataStoreConnection) -> anyhow::Result { conn.transaction_async(|conn| async move { conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL) .await diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 2cd21754f8..5b1163dc8b 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -27,7 +27,8 @@ use crate::db::{ error::{public_error_from_diesel, ErrorHandler}, }; use ::oximeter::types::ProducerRegistry; -use async_bb8_diesel::{AsyncRunQueryDsl, ConnectionManager}; +use anyhow::{anyhow, bail, Context}; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::pg::Pg; use diesel::prelude::*; use diesel::query_builder::{QueryFragment, QueryId}; @@ -174,8 +175,8 @@ impl RunnableQuery for T where { } -pub type DataStoreConnection<'a> = - bb8::PooledConnection<'a, ConnectionManager>; +pub type DataStoreConnection = + qorb::claim::Handle>; pub struct DataStore { log: Logger, @@ -207,7 +208,7 @@ impl DataStore { /// Constructs a new Datastore object. /// - /// Only returns if the database schema is compatible with Nexus's known + /// Only returns when the database schema is compatible with Nexus's known /// schema version. pub async fn new( log: &Logger, @@ -241,6 +242,38 @@ impl DataStore { Ok(datastore) } + /// Constructs a new Datastore, failing if the schema version does not match + /// this program's expected version + pub async fn new_failfast( + log: &Logger, + pool: Arc, + ) -> Result { + let datastore = + Self::new_unchecked(log.new(o!("component" => "datastore")), pool) + .map_err(|e| anyhow!("{}", e))?; + const EXPECTED_VERSION: SemverVersion = nexus_db_model::SCHEMA_VERSION; + let (found_version, found_target) = datastore + .database_schema_version() + .await + .context("loading database schema version")?; + + if let Some(found_target) = found_target { + bail!( + "database schema check failed: apparently mid-upgrade \ + (found_target = {found_target})" + ); + } + + if found_version != EXPECTED_VERSION { + bail!( + "database schema check failed: \ + expected {EXPECTED_VERSION}, found {found_version}", + ); + } + + Ok(datastore) + } + pub fn register_producers(&self, registry: &ProducerRegistry) { registry .register_producer( @@ -279,8 +312,7 @@ impl DataStore { opctx: &OpContext, ) -> Result { opctx.authorize(authz::Action::Query, &authz::DATABASE).await?; - let pool = self.pool.pool(); - let connection = pool.get().await.map_err(|err| { + let connection = self.pool.claim().await.map_err(|err| { Error::unavail(&format!("Failed to access DB connection: {err}")) })?; Ok(connection) @@ -294,7 +326,7 @@ impl DataStore { pub(super) async fn pool_connection_unauthorized( &self, ) -> Result { - let connection = self.pool.pool().get().await.map_err(|err| { + let connection = self.pool.claim().await.map_err(|err| { Error::unavail(&format!("Failed to access DB connection: {err}")) })?; Ok(connection) @@ -399,10 +431,10 @@ mod test { use crate::db::identity::Asset; use crate::db::lookup::LookupPath; use crate::db::model::{ - BlockSize, ConsoleSession, Dataset, DatasetKind, ExternalIp, - PhysicalDisk, PhysicalDiskKind, PhysicalDiskPolicy, PhysicalDiskState, - Project, Rack, Region, SiloUser, SledBaseboard, SledSystemHardware, - SledUpdate, SshKey, Zpool, + BlockSize, ConsoleSession, Dataset, ExternalIp, PhysicalDisk, + PhysicalDiskKind, PhysicalDiskPolicy, PhysicalDiskState, Project, Rack, + Region, SiloUser, SledBaseboard, SledSystemHardware, SledUpdate, + SshKey, Zpool, }; use crate::db::queries::vpc_subnet::InsertVpcSubnetQuery; use chrono::{Duration, Utc}; @@ -418,6 +450,7 @@ mod test { use omicron_common::api::external::{ ByteCount, Error, IdentityMetadataCreateParams, LookupType, Name, }; + use omicron_common::api::internal::shared::DatasetKind; use omicron_test_utils::dev; use omicron_uuid_kinds::CollectionUuid; use omicron_uuid_kinds::GenericUuid; @@ -1587,7 +1620,7 @@ mod test { dev::test_setup_log("test_queries_do_not_require_full_table_scan"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); let datastore = DataStore::new(&logctx.log, Arc::new(pool), None).await.unwrap(); let conn = datastore.pool_connection_for_tests().await.unwrap(); @@ -1632,7 +1665,7 @@ mod test { let logctx = dev::test_setup_log("test_sled_ipv6_address_allocation"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(db::Pool::new(&logctx.log, &cfg)); + let pool = Arc::new(db::Pool::new_single_host(&logctx.log, &cfg)); let datastore = Arc::new(DataStore::new(&logctx.log, pool, None).await.unwrap()); let opctx = OpContext::for_tests( diff --git a/nexus/db-queries/src/db/datastore/probe.rs b/nexus/db-queries/src/db/datastore/probe.rs index f3e0614552..434bf25760 100644 --- a/nexus/db-queries/src/db/datastore/probe.rs +++ b/nexus/db-queries/src/db/datastore/probe.rs @@ -62,7 +62,7 @@ impl super::DataStore { use db::schema::probe::dsl; use db::schema::vpc_subnet::dsl as vpc_subnet_dsl; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let probes = match pagparams { PaginatedBy::Id(pagparams) => { @@ -77,7 +77,7 @@ impl super::DataStore { .filter(dsl::project_id.eq(authz_project.id())) .filter(dsl::time_deleted.is_null()) .select(Probe::as_select()) - .load_async(&*pool) + .load_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; @@ -99,7 +99,7 @@ impl super::DataStore { let db_subnet = vpc_subnet_dsl::vpc_subnet .filter(vpc_subnet_dsl::id.eq(interface.subnet_id)) .select(VpcSubnet::as_select()) - .first_async(&*pool) + .first_async(&*conn) .await .map_err(|e| { public_error_from_diesel(e, ErrorHandler::Server) @@ -126,7 +126,7 @@ impl super::DataStore { &self, opctx: &OpContext, probe: &Probe, - pool: &DataStoreConnection<'_>, + conn: &DataStoreConnection, ) -> LookupResult { use db::schema::vpc_subnet::dsl as vpc_subnet_dsl; @@ -143,7 +143,7 @@ impl super::DataStore { let db_subnet = vpc_subnet_dsl::vpc_subnet .filter(vpc_subnet_dsl::id.eq(interface.subnet_id)) .select(VpcSubnet::as_select()) - .first_async(&**pool) + .first_async(&**conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; @@ -172,20 +172,20 @@ impl super::DataStore { ) -> ListResultVec { use db::schema::probe::dsl; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let probes = paginated(dsl::probe, dsl::id, pagparams) .filter(dsl::time_deleted.is_null()) .filter(dsl::sled.eq(sled)) .select(Probe::as_select()) - .load_async(&*pool) + .load_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; let mut result = Vec::with_capacity(probes.len()); for probe in probes.into_iter() { - result.push(self.resolve_probe_info(opctx, &probe, &pool).await?); + result.push(self.resolve_probe_info(opctx, &probe, &conn).await?); } Ok(result) @@ -200,7 +200,7 @@ impl super::DataStore { ) -> LookupResult { use db::schema::probe; use db::schema::probe::dsl; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let name_or_id = name_or_id.clone(); @@ -211,7 +211,7 @@ impl super::DataStore { .filter(probe::project_id.eq(authz_project.id())) .select(Probe::as_select()) .limit(1) - .first_async::(&*pool) + .first_async::(&*conn) .await .map_err(|e| { public_error_from_diesel( @@ -227,7 +227,7 @@ impl super::DataStore { .filter(probe::project_id.eq(authz_project.id())) .select(Probe::as_select()) .limit(1) - .first_async::(&*pool) + .first_async::(&*conn) .await .map_err(|e| { public_error_from_diesel( @@ -240,7 +240,7 @@ impl super::DataStore { }), }?; - self.resolve_probe_info(opctx, &probe, &pool).await + self.resolve_probe_info(opctx, &probe, &conn).await } /// Add a probe to the data store. @@ -253,7 +253,7 @@ impl super::DataStore { ) -> CreateResult { //TODO in transaction use db::schema::probe::dsl; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let _eip = self .allocate_probe_ephemeral_ip( @@ -306,7 +306,7 @@ impl super::DataStore { let result = diesel::insert_into(dsl::probe) .values(probe.clone()) .returning(Probe::as_returning()) - .get_result_async(&*pool) + .get_result_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; @@ -322,7 +322,7 @@ impl super::DataStore { ) -> DeleteResult { use db::schema::probe; use db::schema::probe::dsl; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let name_or_id = name_or_id.clone(); @@ -334,7 +334,7 @@ impl super::DataStore { .filter(probe::project_id.eq(authz_project.id())) .select(probe::id) .limit(1) - .first_async::(&*pool) + .first_async::(&*conn) .await .map_err(|e| { public_error_from_diesel(e, ErrorHandler::Server) @@ -350,7 +350,7 @@ impl super::DataStore { .filter(dsl::id.eq(id)) .filter(dsl::project_id.eq(authz_project.id())) .set(dsl::time_deleted.eq(Utc::now())) - .execute_async(&*pool) + .execute_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; diff --git a/nexus/db-queries/src/db/datastore/pub_test_utils.rs b/nexus/db-queries/src/db/datastore/pub_test_utils.rs index 93a172bd15..bcf6a6c80f 100644 --- a/nexus/db-queries/src/db/datastore/pub_test_utils.rs +++ b/nexus/db-queries/src/db/datastore/pub_test_utils.rs @@ -29,7 +29,7 @@ pub async fn datastore_test( use crate::authn; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(db::Pool::new(&logctx.log, &cfg)); + let pool = Arc::new(db::Pool::new_single_host(&logctx.log, &cfg)); let datastore = Arc::new(DataStore::new(&logctx.log, pool, None).await.unwrap()); diff --git a/nexus/db-queries/src/db/datastore/vmm.rs b/nexus/db-queries/src/db/datastore/vmm.rs index 14c3405a70..089a2914be 100644 --- a/nexus/db-queries/src/db/datastore/vmm.rs +++ b/nexus/db-queries/src/db/datastore/vmm.rs @@ -5,7 +5,6 @@ //! [`DataStore`] helpers for working with VMM records. use super::DataStore; -use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; @@ -40,8 +39,13 @@ use uuid::Uuid; /// The result of an [`DataStore::vmm_and_migration_update_runtime`] call, /// indicating which records were updated. -#[derive(Copy, Clone, Debug)] +#[derive(Clone, Debug)] pub struct VmmStateUpdateResult { + /// The VMM record that the update query found and possibly updated. + /// + /// NOTE: This is the record prior to the update! + pub found_vmm: Vmm, + /// `true` if the VMM record was updated, `false` otherwise. pub vmm_updated: bool, @@ -108,14 +112,10 @@ impl DataStore { pub async fn vmm_fetch( &self, opctx: &OpContext, - authz_instance: &authz::Instance, vmm_id: &PropolisUuid, ) -> LookupResult { - opctx.authorize(authz::Action::Read, authz_instance).await?; - let vmm = dsl::vmm .filter(dsl::id.eq(vmm_id.into_untyped_uuid())) - .filter(dsl::instance_id.eq(authz_instance.id())) .filter(dsl::time_deleted.is_null()) .select(Vmm::as_select()) .get_result_async(&*self.pool_connection_authorized(opctx).await?) @@ -233,13 +233,21 @@ impl DataStore { .transaction(&conn, |conn| { let err = err.clone(); async move { - let vmm_updated = self + let vmm_update_result = self .vmm_update_runtime_on_connection( &conn, &vmm_id, new_runtime, ) - .await.map(|r| match r.status { UpdateStatus::Updated => true, UpdateStatus::NotUpdatedButExists => false })?; + .await?; + + + let found_vmm = vmm_update_result.found; + let vmm_updated = match vmm_update_result.status { + UpdateStatus::Updated => true, + UpdateStatus::NotUpdatedButExists => false + }; + let migration_out_updated = match migration_out { Some(migration) => { let r = self.migration_update_source_on_connection( @@ -287,6 +295,7 @@ impl DataStore { None => false, }; Ok(VmmStateUpdateResult { + found_vmm, vmm_updated, migration_in_updated, migration_out_updated, diff --git a/nexus/db-queries/src/db/explain.rs b/nexus/db-queries/src/db/explain.rs index 24fd993040..52844c204f 100644 --- a/nexus/db-queries/src/db/explain.rs +++ b/nexus/db-queries/src/db/explain.rs @@ -124,8 +124,7 @@ mod test { } async fn create_schema(pool: &db::Pool) { - pool.pool() - .get() + pool.claim() .await .unwrap() .batch_execute_async( @@ -145,8 +144,8 @@ mod test { let logctx = dev::test_setup_log("test_explain_async"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); - let conn = pool.pool().get().await.unwrap(); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); + let conn = pool.claim().await.unwrap(); create_schema(&pool).await; @@ -170,8 +169,8 @@ mod test { let logctx = dev::test_setup_log("test_explain_full_table_scan"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); - let conn = pool.pool().get().await.unwrap(); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); + let conn = pool.claim().await.unwrap(); create_schema(&pool).await; diff --git a/nexus/db-queries/src/db/pagination.rs b/nexus/db-queries/src/db/pagination.rs index 4fc1cf5966..9920440ade 100644 --- a/nexus/db-queries/src/db/pagination.rs +++ b/nexus/db-queries/src/db/pagination.rs @@ -354,7 +354,7 @@ mod test { async fn populate_users(pool: &db::Pool, values: &Vec<(i64, i64)>) { use schema::test_users::dsl; - let conn = pool.pool().get().await.unwrap(); + let conn = pool.claim().await.unwrap(); // The indexes here work around the check that prevents full table // scans. @@ -392,7 +392,7 @@ mod test { pool: &db::Pool, query: BoxedQuery, ) -> Vec { - let conn = pool.pool().get().await.unwrap(); + let conn = pool.claim().await.unwrap(); query.select(User::as_select()).load_async(&*conn).await.unwrap() } @@ -402,7 +402,7 @@ mod test { dev::test_setup_log("test_paginated_single_column_ascending"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); use schema::test_users::dsl; @@ -437,7 +437,7 @@ mod test { dev::test_setup_log("test_paginated_single_column_descending"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); use schema::test_users::dsl; @@ -472,7 +472,7 @@ mod test { dev::test_setup_log("test_paginated_multicolumn_ascending"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); use schema::test_users::dsl; @@ -526,7 +526,7 @@ mod test { dev::test_setup_log("test_paginated_multicolumn_descending"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = db::Pool::new(&logctx.log, &cfg); + let pool = db::Pool::new_single_host(&logctx.log, &cfg); use schema::test_users::dsl; diff --git a/nexus/db-queries/src/db/pool.rs b/nexus/db-queries/src/db/pool.rs index 497c8d97c5..dccee6fa3f 100644 --- a/nexus/db-queries/src/db/pool.rs +++ b/nexus/db-queries/src/db/pool.rs @@ -3,108 +3,155 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. //! Database connection pooling -// This whole thing is a placeholder for prototyping. -// -// TODO-robustness TODO-resilience We will want to carefully think about the -// connection pool that we use and its parameters. It's not clear from the -// survey so far whether an existing module is suitable for our purposes. See -// the Cueball Internals document for details on the sorts of behaviors we'd -// like here. Even if by luck we stick with bb8, we definitely want to think -// through the various parameters. -// -// Notes about bb8's behavior: -// * When the database is completely offline, and somebody wants a connection, -// it still waits for the connection timeout before giving up. That seems -// like not what we want. (To be clear, this is a failure mode where we know -// the database is offline, not one where it's partitioned and we can't tell.) -// * Although the `build_unchecked()` builder allows the pool to start up with -// no connections established (good), it also _seems_ to not establish any -// connections even when it could, resulting in a latency bubble for the first -// operation after startup. That's not what we're looking for. -// // TODO-design Need TLS support (the types below hardcode NoTls). use super::Config as DbConfig; -use async_bb8_diesel::ConnectionError; -use async_bb8_diesel::ConnectionManager; +use crate::db::pool_connection::{DieselPgConnector, DieselPgConnectorArgs}; + +use qorb::backend; +use qorb::policy::Policy; +use qorb::resolver::{AllBackends, Resolver}; +use qorb::resolvers::dns::{DnsResolver, DnsResolverConfig}; +use qorb::service; +use slog::Logger; +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::sync::watch; pub use super::pool_connection::DbConnection; +type QorbConnection = async_bb8_diesel::Connection; +type QorbPool = qorb::pool::Pool; + /// Wrapper around a database connection pool. /// /// Expected to be used as the primary interface to the database. pub struct Pool { - pool: bb8::Pool>, + inner: QorbPool, } -impl Pool { - pub fn new(log: &slog::Logger, db_config: &DbConfig) -> Self { - // Make sure diesel-dtrace's USDT probes are enabled. - usdt::register_probes().expect("Failed to register USDT DTrace probes"); - Self::new_builder(log, db_config, bb8::Builder::new()) - } +// Provides an alternative to the DNS resolver for cases where we want to +// contact the database without performing resolution. +struct SingleHostResolver { + tx: watch::Sender, +} - pub fn new_failfast_for_tests( - log: &slog::Logger, - db_config: &DbConfig, - ) -> Self { - Self::new_builder( - log, - db_config, - bb8::Builder::new() - .connection_timeout(std::time::Duration::from_millis(1)), - ) +impl SingleHostResolver { + fn new(config: &DbConfig) -> Self { + let backends = Arc::new(BTreeMap::from([( + backend::Name::new("singleton"), + backend::Backend { address: config.url.address() }, + )])); + let (tx, _rx) = watch::channel(backends.clone()); + Self { tx } } +} - fn new_builder( - log: &slog::Logger, - db_config: &DbConfig, - builder: bb8::Builder>, - ) -> Self { - let url = db_config.url.url(); - let log = log.new(o!( - "database_url" => url.clone(), - "component" => "db::Pool" - )); - info!(&log, "database connection pool"); - let error_sink = LoggingErrorSink::new(log); - let manager = ConnectionManager::::new(url); - let pool = builder - .connection_customizer(Box::new( - super::pool_connection::ConnectionCustomizer::new(), - )) - .error_sink(Box::new(error_sink)) - .build_unchecked(manager); - Pool { pool } +impl Resolver for SingleHostResolver { + fn monitor(&mut self) -> watch::Receiver { + self.tx.subscribe() } +} - /// Returns a reference to the underlying pool. - pub fn pool(&self) -> &bb8::Pool> { - &self.pool - } +fn make_dns_resolver( + bootstrap_dns: Vec, +) -> qorb::resolver::BoxedResolver { + Box::new(DnsResolver::new( + service::Name(internal_dns::ServiceName::Cockroach.srv_name()), + bootstrap_dns, + DnsResolverConfig { + hardcoded_ttl: Some(tokio::time::Duration::MAX), + ..Default::default() + }, + )) } -#[derive(Clone, Debug)] -struct LoggingErrorSink { - log: slog::Logger, +fn make_single_host_resolver( + config: &DbConfig, +) -> qorb::resolver::BoxedResolver { + Box::new(SingleHostResolver::new(config)) } -impl LoggingErrorSink { - fn new(log: slog::Logger) -> LoggingErrorSink { - LoggingErrorSink { log } - } +fn make_postgres_connector( + log: &Logger, +) -> qorb::backend::SharedConnector { + // Create postgres connections. + // + // We're currently relying on the DieselPgConnector doing the following: + // - Disallowing full table scans in its implementation of "on_acquire" + // - Creating async_bb8_diesel connections that also wrap DTraceConnections. + let user = "root"; + let db = "omicron"; + let args = vec![("sslmode", "disable")]; + Arc::new(DieselPgConnector::new( + log, + DieselPgConnectorArgs { user, db, args }, + )) } -impl bb8::ErrorSink for LoggingErrorSink { - fn sink(&self, error: ConnectionError) { - error!( - &self.log, - "database connection error"; - "error_message" => #%error - ); +impl Pool { + /// Creates a new qorb-backed connection pool to the database. + /// + /// Creating this pool does not necessarily wait for connections to become + /// available, as backends may shift over time. + pub fn new(log: &Logger, bootstrap_dns: Vec) -> Self { + // Make sure diesel-dtrace's USDT probes are enabled. + usdt::register_probes().expect("Failed to register USDT DTrace probes"); + + let resolver = make_dns_resolver(bootstrap_dns); + let connector = make_postgres_connector(log); + + let policy = Policy::default(); + Pool { inner: qorb::pool::Pool::new(resolver, connector, policy) } + } + + /// Creates a new qorb-backed connection pool to a single instance of the + /// database. + /// + /// This is intended for tests that want to skip DNS resolution, relying + /// on a single instance of the database. + /// + /// In production, [Self::new] should be preferred. + pub fn new_single_host(log: &Logger, db_config: &DbConfig) -> Self { + // Make sure diesel-dtrace's USDT probes are enabled. + usdt::register_probes().expect("Failed to register USDT DTrace probes"); + + let resolver = make_single_host_resolver(db_config); + let connector = make_postgres_connector(log); + + let policy = Policy::default(); + Pool { inner: qorb::pool::Pool::new(resolver, connector, policy) } + } + + /// Creates a new qorb-backed connection pool which returns an error + /// if claims are not available within one millisecond. + /// + /// This is intended for test-only usage, in particular for tests where + /// claim requests should rapidly return errors when a backend has been + /// intentionally disabled. + #[cfg(any(test, feature = "testing"))] + pub fn new_single_host_failfast( + log: &Logger, + db_config: &DbConfig, + ) -> Self { + // Make sure diesel-dtrace's USDT probes are enabled. + usdt::register_probes().expect("Failed to register USDT DTrace probes"); + + let resolver = make_single_host_resolver(db_config); + let connector = make_postgres_connector(log); + + let policy = Policy { + claim_timeout: tokio::time::Duration::from_millis(1), + ..Default::default() + }; + Pool { inner: qorb::pool::Pool::new(resolver, connector, policy) } } - fn boxed_clone(&self) -> Box> { - Box::new(self.clone()) + /// Returns a connection from the pool + pub async fn claim( + &self, + ) -> anyhow::Result> { + Ok(self.inner.claim().await?) } } diff --git a/nexus/db-queries/src/db/pool_connection.rs b/nexus/db-queries/src/db/pool_connection.rs index dae6a0ee51..9a33370a5a 100644 --- a/nexus/db-queries/src/db/pool_connection.rs +++ b/nexus/db-queries/src/db/pool_connection.rs @@ -4,46 +4,139 @@ //! Customization that happens on each connection as they're acquired. +use anyhow::anyhow; +use async_bb8_diesel::AsyncR2D2Connection; use async_bb8_diesel::AsyncSimpleConnection; -use async_bb8_diesel::Connection; -use async_bb8_diesel::ConnectionError; use async_trait::async_trait; -use bb8::CustomizeConnection; +use diesel::Connection; use diesel::PgConnection; use diesel_dtrace::DTraceConnection; +use qorb::backend::{self, Backend, Error}; +use slog::Logger; +use url::Url; pub type DbConnection = DTraceConnection; pub const DISALLOW_FULL_TABLE_SCAN_SQL: &str = "set disallow_full_table_scans = on; set large_full_scan_rows = 0;"; -/// A customizer for all new connections made to CockroachDB, from Diesel. -#[derive(Debug)] -pub(crate) struct ConnectionCustomizer {} +/// A [backend::Connector] which provides access to [PgConnection]. +pub(crate) struct DieselPgConnector { + log: Logger, + user: String, + db: String, + args: Vec<(String, String)>, +} + +pub(crate) struct DieselPgConnectorArgs<'a> { + pub(crate) user: &'a str, + pub(crate) db: &'a str, + pub(crate) args: Vec<(&'a str, &'a str)>, +} -impl ConnectionCustomizer { - pub(crate) fn new() -> Self { - Self {} +impl DieselPgConnector { + /// Creates a new "connector" to a database, which + /// swaps out the IP address at runtime depending on the selected backend. + /// + /// Format of the url is: + /// + /// - postgresql://{user}@{address}/{db} + /// + /// Or, if arguments are supplied: + /// + /// - postgresql://{user}@{address}/{db}?{args} + pub(crate) fn new(log: &Logger, args: DieselPgConnectorArgs<'_>) -> Self { + let DieselPgConnectorArgs { user, db, args } = args; + Self { + log: log.clone(), + user: user.to_string(), + db: db.to_string(), + args: args + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + } } - async fn disallow_full_table_scans( + fn to_url( &self, - conn: &mut Connection, - ) -> Result<(), ConnectionError> { - conn.batch_execute_async(DISALLOW_FULL_TABLE_SCAN_SQL).await?; - Ok(()) + address: std::net::SocketAddr, + ) -> Result { + let user = &self.user; + let db = &self.db; + let mut url = + Url::parse(&format!("postgresql://{user}@{address}/{db}"))?; + + for (k, v) in &self.args { + url.query_pairs_mut().append_pair(k, v); + } + + Ok(url.as_str().to_string()) } } #[async_trait] -impl CustomizeConnection, ConnectionError> - for ConnectionCustomizer -{ +impl backend::Connector for DieselPgConnector { + type Connection = async_bb8_diesel::Connection; + + async fn connect( + &self, + backend: &Backend, + ) -> Result { + let url = self.to_url(backend.address).map_err(Error::Other)?; + + let conn = tokio::task::spawn_blocking(move || { + let pg_conn = DbConnection::establish(&url) + .map_err(|e| Error::Other(anyhow!(e)))?; + Ok::<_, Error>(async_bb8_diesel::Connection::new(pg_conn)) + }) + .await + .expect("Task panicked establishing connection") + .map_err(|e| { + warn!( + self.log, + "Failed to make connection"; + "error" => e.to_string(), + "backend" => backend.address, + ); + e + })?; + Ok(conn) + } + async fn on_acquire( &self, - conn: &mut Connection, - ) -> Result<(), ConnectionError> { - self.disallow_full_table_scans(conn).await?; + conn: &mut Self::Connection, + ) -> Result<(), Error> { + conn.batch_execute_async(DISALLOW_FULL_TABLE_SCAN_SQL).await.map_err( + |e| { + warn!( + self.log, + "Failed on_acquire execution"; + "error" => e.to_string() + ); + Error::Other(anyhow!(e)) + }, + )?; Ok(()) } + + async fn is_valid(&self, conn: &mut Self::Connection) -> Result<(), Error> { + let is_broken = conn.is_broken_async().await; + if is_broken { + warn!( + self.log, + "Failed is_valid check; connection known to be broken" + ); + return Err(Error::Other(anyhow!("Connection broken"))); + } + conn.ping_async().await.map_err(|e| { + warn!( + self.log, + "Failed is_valid check; connection failed ping"; + "error" => e.to_string() + ); + Error::Other(anyhow!(e)) + }) + } } diff --git a/nexus/db-queries/src/db/queries/external_ip.rs b/nexus/db-queries/src/db/queries/external_ip.rs index 7ea44b33fb..4d752d451b 100644 --- a/nexus/db-queries/src/db/queries/external_ip.rs +++ b/nexus/db-queries/src/db/queries/external_ip.rs @@ -918,7 +918,8 @@ mod tests { crate::db::datastore::test_utils::datastore_test(&logctx, &db) .await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg)); + let pool = + Arc::new(crate::db::Pool::new_single_host(&logctx.log, &cfg)); let db_datastore = Arc::new( crate::db::DataStore::new(&logctx.log, Arc::clone(&pool), None) .await diff --git a/nexus/db-queries/src/db/queries/next_item.rs b/nexus/db-queries/src/db/queries/next_item.rs index 769c891349..658d151a5b 100644 --- a/nexus/db-queries/src/db/queries/next_item.rs +++ b/nexus/db-queries/src/db/queries/next_item.rs @@ -616,7 +616,7 @@ mod tests { } async fn setup_test_schema(pool: &db::Pool) { - let connection = pool.pool().get().await.unwrap(); + let connection = pool.claim().await.unwrap(); (*connection) .batch_execute_async( "CREATE SCHEMA IF NOT EXISTS test_schema; \ @@ -708,8 +708,9 @@ mod tests { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg)); - let conn = pool.pool().get().await.unwrap(); + let pool = + Arc::new(crate::db::Pool::new_single_host(&logctx.log, &cfg)); + let conn = pool.claim().await.unwrap(); // We're going to operate on a separate table, for simplicity. setup_test_schema(&pool).await; @@ -770,8 +771,9 @@ mod tests { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg)); - let conn = pool.pool().get().await.unwrap(); + let pool = + Arc::new(crate::db::Pool::new_single_host(&logctx.log, &cfg)); + let conn = pool.claim().await.unwrap(); // We're going to operate on a separate table, for simplicity. setup_test_schema(&pool).await; diff --git a/nexus/db-queries/src/db/queries/region_allocation.rs b/nexus/db-queries/src/db/queries/region_allocation.rs index 7cf378d53b..dbf37fda2e 100644 --- a/nexus/db-queries/src/db/queries/region_allocation.rs +++ b/nexus/db-queries/src/db/queries/region_allocation.rs @@ -507,8 +507,8 @@ mod test { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = crate::db::Pool::new(&logctx.log, &cfg); - let conn = pool.pool().get().await.unwrap(); + let pool = crate::db::Pool::new_single_host(&logctx.log, &cfg); + let conn = pool.claim().await.unwrap(); let volume_id = Uuid::new_v4(); let params = RegionParameters { diff --git a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs index 902d955a79..9d2ed04c85 100644 --- a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs +++ b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs @@ -568,8 +568,8 @@ mod test { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = crate::db::Pool::new(&logctx.log, &cfg); - let conn = pool.pool().get().await.unwrap(); + let pool = crate::db::Pool::new_single_host(&logctx.log, &cfg); + let conn = pool.claim().await.unwrap(); let id = Uuid::nil(); let project_id = Uuid::nil(); @@ -597,8 +597,8 @@ mod test { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = crate::db::Pool::new(&logctx.log, &cfg); - let conn = pool.pool().get().await.unwrap(); + let pool = crate::db::Pool::new_single_host(&logctx.log, &cfg); + let conn = pool.claim().await.unwrap(); let id = Uuid::nil(); let project_id = Uuid::nil(); @@ -624,8 +624,8 @@ mod test { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = crate::db::Pool::new(&logctx.log, &cfg); - let conn = pool.pool().get().await.unwrap(); + let pool = crate::db::Pool::new_single_host(&logctx.log, &cfg); + let conn = pool.claim().await.unwrap(); let id = InstanceUuid::nil(); let project_id = Uuid::nil(); @@ -650,8 +650,8 @@ mod test { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = crate::db::Pool::new(&logctx.log, &cfg); - let conn = pool.pool().get().await.unwrap(); + let pool = crate::db::Pool::new_single_host(&logctx.log, &cfg); + let conn = pool.claim().await.unwrap(); let id = InstanceUuid::nil(); let project_id = Uuid::nil(); diff --git a/nexus/db-queries/src/db/queries/vpc_subnet.rs b/nexus/db-queries/src/db/queries/vpc_subnet.rs index 8cbf4495ca..85c771c050 100644 --- a/nexus/db-queries/src/db/queries/vpc_subnet.rs +++ b/nexus/db-queries/src/db/queries/vpc_subnet.rs @@ -313,8 +313,9 @@ mod test { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg)); - let conn = pool.pool().get().await.unwrap(); + let pool = + Arc::new(crate::db::Pool::new_single_host(&logctx.log, &cfg)); + let conn = pool.claim().await.unwrap(); let explain = query.explain_async(&conn).await.unwrap(); println!("{explain}"); db.cleanup().await.unwrap(); @@ -352,7 +353,8 @@ mod test { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg)); + let pool = + Arc::new(crate::db::Pool::new_single_host(&logctx.log, &cfg)); let db_datastore = Arc::new( crate::db::DataStore::new(&log, Arc::clone(&pool), None) .await @@ -544,7 +546,8 @@ mod test { let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(crate::db::Pool::new(&logctx.log, &cfg)); + let pool = + Arc::new(crate::db::Pool::new_single_host(&logctx.log, &cfg)); let db_datastore = Arc::new( crate::db::DataStore::new(&log, Arc::clone(&pool), None) .await diff --git a/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql b/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql index 6331770ef5..4e7dde244b 100644 --- a/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_distinct_sleds.sql @@ -270,7 +270,8 @@ WITH dataset.ip, dataset.port, dataset.kind, - dataset.size_used + dataset.size_used, + dataset.zone_name ) ( SELECT @@ -284,6 +285,7 @@ WITH dataset.port, dataset.kind, dataset.size_used, + dataset.zone_name, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -310,6 +312,7 @@ UNION updated_datasets.port, updated_datasets.kind, updated_datasets.size_used, + updated_datasets.zone_name, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/db-queries/tests/output/region_allocate_random_sleds.sql b/nexus/db-queries/tests/output/region_allocate_random_sleds.sql index e713121d34..b2c164a6d9 100644 --- a/nexus/db-queries/tests/output/region_allocate_random_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_random_sleds.sql @@ -268,7 +268,8 @@ WITH dataset.ip, dataset.port, dataset.kind, - dataset.size_used + dataset.size_used, + dataset.zone_name ) ( SELECT @@ -282,6 +283,7 @@ WITH dataset.port, dataset.kind, dataset.size_used, + dataset.zone_name, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -308,6 +310,7 @@ UNION updated_datasets.port, updated_datasets.kind, updated_datasets.size_used, + updated_datasets.zone_name, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql b/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql index 0b8dc4fca6..97ee23f82e 100644 --- a/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_with_snapshot_distinct_sleds.sql @@ -281,7 +281,8 @@ WITH dataset.ip, dataset.port, dataset.kind, - dataset.size_used + dataset.size_used, + dataset.zone_name ) ( SELECT @@ -295,6 +296,7 @@ WITH dataset.port, dataset.kind, dataset.size_used, + dataset.zone_name, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -321,6 +323,7 @@ UNION updated_datasets.port, updated_datasets.kind, updated_datasets.size_used, + updated_datasets.zone_name, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql b/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql index 9ac945f71d..a1cc103594 100644 --- a/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql +++ b/nexus/db-queries/tests/output/region_allocate_with_snapshot_random_sleds.sql @@ -279,7 +279,8 @@ WITH dataset.ip, dataset.port, dataset.kind, - dataset.size_used + dataset.size_used, + dataset.zone_name ) ( SELECT @@ -293,6 +294,7 @@ WITH dataset.port, dataset.kind, dataset.size_used, + dataset.zone_name, old_regions.id, old_regions.time_created, old_regions.time_modified, @@ -319,6 +321,7 @@ UNION updated_datasets.port, updated_datasets.kind, updated_datasets.size_used, + updated_datasets.zone_name, inserted_regions.id, inserted_regions.time_created, inserted_regions.time_modified, diff --git a/nexus/external-api/Cargo.toml b/nexus/external-api/Cargo.toml new file mode 100644 index 0000000000..0875e1f574 --- /dev/null +++ b/nexus/external-api/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "nexus-external-api" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +dropshot.workspace = true +http.workspace = true +hyper.workspace = true +ipnetwork.workspace = true +nexus-types.workspace = true +omicron-common.workspace = true +omicron-workspace-hack.workspace = true +openapiv3.workspace = true +openapi-manager-types.workspace = true +oximeter-types.workspace = true +oxql-types.workspace = true diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt similarity index 100% rename from nexus/tests/output/nexus_tags.txt rename to nexus/external-api/output/nexus_tags.txt diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs new file mode 100644 index 0000000000..669b25145f --- /dev/null +++ b/nexus/external-api/src/lib.rs @@ -0,0 +1,3032 @@ +use std::collections::BTreeMap; + +use anyhow::anyhow; +use dropshot::{ + EmptyScanParams, EndpointTagPolicy, HttpError, HttpResponseAccepted, + HttpResponseCreated, HttpResponseDeleted, HttpResponseFound, + HttpResponseHeaders, HttpResponseOk, HttpResponseSeeOther, + HttpResponseUpdatedNoContent, PaginationParams, Path, Query, + RequestContext, ResultsPage, StreamingBody, TypedBody, + WebsocketChannelResult, WebsocketConnection, +}; +use http::Response; +use hyper::Body; +use ipnetwork::IpNetwork; +use nexus_types::{ + authn::cookies::Cookies, + external_api::{params, shared, views}, +}; +use omicron_common::api::external::{ + http_pagination::{PaginatedById, PaginatedByName, PaginatedByNameOrId}, + *, +}; +use openapi_manager_types::ValidationContext; +use openapiv3::OpenAPI; + +pub const API_VERSION: &str = "20240821.0"; + +// API ENDPOINT FUNCTION NAMING CONVENTIONS +// +// Generally, HTTP resources are grouped within some collection. For a +// relatively simple example: +// +// GET v1/projects (list the projects in the collection) +// POST v1/projects (create a project in the collection) +// GET v1/projects/{project} (look up a project in the collection) +// DELETE v1/projects/{project} (delete a project in the collection) +// PUT v1/projects/{project} (update a project in the collection) +// +// We pick a name for the function that implements a given API entrypoint +// based on how we expect it to appear in the CLI subcommand hierarchy. For +// example: +// +// GET v1/projects -> project_list() +// POST v1/projects -> project_create() +// GET v1/projects/{project} -> project_view() +// DELETE v1/projects/{project} -> project_delete() +// PUT v1/projects/{project} -> project_update() +// +// Note that the path typically uses the entity's plural form while the +// function name uses its singular. +// +// Operations beyond list, create, view, delete, and update should use a +// descriptive noun or verb, again bearing in mind that this will be +// transcribed into the CLI and SDKs: +// +// POST -> instance_reboot +// POST -> instance_stop +// GET -> instance_serial_console +// +// Note that these function names end up in generated OpenAPI spec as the +// operationId for each endpoint, and therefore represent a contract with +// clients. Client generators use operationId to name API methods, so changing +// a function name is a breaking change from a client perspective. + +#[dropshot::api_description { + tag_config = { + allow_other_tags = false, + policy = EndpointTagPolicy::ExactlyOne, + tags = { + "disks" = { + description = "Virtual disks are used to store instance-local data which includes the operating system.", + external_docs = { + url = "http://docs.oxide.computer/api/disks" + } + }, + "floating-ips" = { + description = "Floating IPs allow a project to allocate well-known IPs to instances.", + external_docs = { + url = "http://docs.oxide.computer/api/floating-ips" + } + }, + "hidden" = { + description = "TODO operations that will not ship to customers", + external_docs = { + url = "http://docs.oxide.computer/api" + } + }, + "images" = { + description = "Images are read-only virtual disks that may be used to boot virtual machines.", + external_docs = { + url = "http://docs.oxide.computer/api/images" + } + }, + "instances" = { + description = "Virtual machine instances are the basic unit of computation. These operations are used for provisioning, controlling, and destroying instances.", + external_docs = { + url = "http://docs.oxide.computer/api/instances" + } + }, + "login" = { + description = "Authentication endpoints", + external_docs = { + url = "http://docs.oxide.computer/api/login" + } + }, + "metrics" = { + description = "Silo-scoped metrics", + external_docs = { + url = "http://docs.oxide.computer/api/metrics" + } + }, + "policy" = { + description = "System-wide IAM policy", + external_docs = { + url = "http://docs.oxide.computer/api/policy" + } + }, + "projects" = { + description = "Projects are a grouping of associated resources such as instances and disks within a silo for purposes of billing and access control.", + external_docs = { + url = "http://docs.oxide.computer/api/projects" + } + }, + "roles" = { + description = "Roles are a component of Identity and Access Management (IAM) that allow a user or agent account access to additional permissions.", + external_docs = { + url = "http://docs.oxide.computer/api/roles" + } + }, + "session" = { + description = "Information pertaining to the current session.", + external_docs = { + url = "http://docs.oxide.computer/api/session" + } + }, + "silos" = { + description = "Silos represent a logical partition of users and resources.", + external_docs = { + url = "http://docs.oxide.computer/api/silos" + } + }, + "snapshots" = { + description = "Snapshots of virtual disks at a particular point in time.", + external_docs = { + url = "http://docs.oxide.computer/api/snapshots" + } + }, + "vpcs" = { + description = "Virtual Private Clouds (VPCs) provide isolated network environments for managing and deploying services.", + external_docs = { + url = "http://docs.oxide.computer/api/vpcs" + } + }, + "system/probes" = { + description = "Probes for testing network connectivity", + external_docs = { + url = "http://docs.oxide.computer/api/probes" + } + }, + "system/status" = { + description = "Endpoints related to system health", + external_docs = { + url = "http://docs.oxide.computer/api/system-status" + } + }, + "system/hardware" = { + description = "These operations pertain to hardware inventory and management. Racks are the unit of expansion of an Oxide deployment. Racks are in turn composed of sleds, switches, power supplies, and a cabled backplane.", + external_docs = { + url = "http://docs.oxide.computer/api/system-hardware" + } + }, + "system/metrics" = { + description = "Metrics provide insight into the operation of the Oxide deployment. These include telemetry on hardware and software components that can be used to understand the current state as well as to diagnose issues.", + external_docs = { + url = "http://docs.oxide.computer/api/system-metrics" + } + }, + "system/networking" = { + description = "This provides rack-level network configuration.", + external_docs = { + url = "http://docs.oxide.computer/api/system-networking" + } + }, + "system/silos" = { + description = "Silos represent a logical partition of users and resources.", + external_docs = { + url = "http://docs.oxide.computer/api/system-silos" + } + } + } + } +}] +pub trait NexusExternalApi { + type Context; + + /// Ping API + /// + /// Always responds with Ok if it responds at all. + #[endpoint { + method = GET, + path = "/v1/ping", + tags = ["system/status"], + }] + async fn ping( + _rqctx: RequestContext, + ) -> Result, HttpError> { + Ok(HttpResponseOk(views::Ping { status: views::PingStatus::Ok })) + } + + /// Fetch top-level IAM policy + #[endpoint { + method = GET, + path = "/v1/system/policy", + tags = ["policy"], + }] + async fn system_policy_view( + rqctx: RequestContext, + ) -> Result>, HttpError>; + + /// Update top-level IAM policy + #[endpoint { + method = PUT, + path = "/v1/system/policy", + tags = ["policy"], + }] + async fn system_policy_update( + rqctx: RequestContext, + new_policy: TypedBody>, + ) -> Result>, HttpError>; + + /// Fetch current silo's IAM policy + #[endpoint { + method = GET, + path = "/v1/policy", + tags = ["silos"], + }] + async fn policy_view( + rqctx: RequestContext, + ) -> Result>, HttpError>; + + /// Update current silo's IAM policy + #[endpoint { + method = PUT, + path = "/v1/policy", + tags = ["silos"], + }] + async fn policy_update( + rqctx: RequestContext, + new_policy: TypedBody>, + ) -> Result>, HttpError>; + + /// Fetch resource utilization for user's current silo + #[endpoint { + method = GET, + path = "/v1/utilization", + tags = ["silos"], + }] + async fn utilization_view( + rqctx: RequestContext, + ) -> Result, HttpError>; + + /// Fetch current utilization for given silo + #[endpoint { + method = GET, + path = "/v1/system/utilization/silos/{silo}", + tags = ["system/silos"], + }] + async fn silo_utilization_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// List current utilization state for all silos + #[endpoint { + method = GET, + path = "/v1/system/utilization/silos", + tags = ["system/silos"], + }] + async fn silo_utilization_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Lists resource quotas for all silos + #[endpoint { + method = GET, + path = "/v1/system/silo-quotas", + tags = ["system/silos"], + }] + async fn system_quotas_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Fetch resource quotas for silo + #[endpoint { + method = GET, + path = "/v1/system/silos/{silo}/quotas", + tags = ["system/silos"], + }] + async fn silo_quotas_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Update resource quotas for silo + /// + /// If a quota value is not specified, it will remain unchanged. + #[endpoint { + method = PUT, + path = "/v1/system/silos/{silo}/quotas", + tags = ["system/silos"], + }] + async fn silo_quotas_update( + rqctx: RequestContext, + path_params: Path, + new_quota: TypedBody, + ) -> Result, HttpError>; + + /// List silos + /// + /// Lists silos that are discoverable based on the current permissions. + #[endpoint { + method = GET, + path = "/v1/system/silos", + tags = ["system/silos"], + }] + async fn silo_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Create a silo + #[endpoint { + method = POST, + path = "/v1/system/silos", + tags = ["system/silos"], + }] + async fn silo_create( + rqctx: RequestContext, + new_silo_params: TypedBody, + ) -> Result, HttpError>; + + /// Fetch silo + /// + /// Fetch silo by name or ID. + #[endpoint { + method = GET, + path = "/v1/system/silos/{silo}", + tags = ["system/silos"], + }] + async fn silo_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// List IP pools linked to silo + /// + /// Linked IP pools are available to users in the specified silo. A silo + /// can have at most one default pool. IPs are allocated from the default + /// pool when users ask for one without specifying a pool. + #[endpoint { + method = GET, + path = "/v1/system/silos/{silo}/ip-pools", + tags = ["system/silos"], + }] + async fn silo_ip_pool_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError>; + + /// Delete a silo + /// + /// Delete a silo by name or ID. + #[endpoint { + method = DELETE, + path = "/v1/system/silos/{silo}", + tags = ["system/silos"], + }] + async fn silo_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + /// Fetch silo IAM policy + #[endpoint { + method = GET, + path = "/v1/system/silos/{silo}/policy", + tags = ["system/silos"], + }] + async fn silo_policy_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError>; + + /// Update silo IAM policy + #[endpoint { + method = PUT, + path = "/v1/system/silos/{silo}/policy", + tags = ["system/silos"], + }] + async fn silo_policy_update( + rqctx: RequestContext, + path_params: Path, + new_policy: TypedBody>, + ) -> Result>, HttpError>; + + // Silo-specific user endpoints + + /// List built-in (system) users in silo + #[endpoint { + method = GET, + path = "/v1/system/users", + tags = ["system/silos"], + }] + async fn silo_user_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Fetch built-in (system) user + #[endpoint { + method = GET, + path = "/v1/system/users/{user_id}", + tags = ["system/silos"], + }] + async fn silo_user_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + // Silo identity providers + + /// List a silo's IdP's name + #[endpoint { + method = GET, + path = "/v1/system/identity-providers", + tags = ["system/silos"], + }] + async fn silo_identity_provider_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + // Silo SAML identity providers + + /// Create SAML IdP + #[endpoint { + method = POST, + path = "/v1/system/identity-providers/saml", + tags = ["system/silos"], + }] + async fn saml_identity_provider_create( + rqctx: RequestContext, + query_params: Query, + new_provider: TypedBody, + ) -> Result, HttpError>; + + /// Fetch SAML IdP + #[endpoint { + method = GET, + path = "/v1/system/identity-providers/saml/{provider}", + tags = ["system/silos"], + }] + async fn saml_identity_provider_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + // TODO: no DELETE for identity providers? + + // "Local" Identity Provider + + /// Create user + /// + /// Users can only be created in Silos with `provision_type` == `Fixed`. + /// Otherwise, Silo users are just-in-time (JIT) provisioned when a user + /// first logs in using an external Identity Provider. + #[endpoint { + method = POST, + path = "/v1/system/identity-providers/local/users", + tags = ["system/silos"], + }] + async fn local_idp_user_create( + rqctx: RequestContext, + query_params: Query, + new_user_params: TypedBody, + ) -> Result, HttpError>; + + /// Delete user + #[endpoint { + method = DELETE, + path = "/v1/system/identity-providers/local/users/{user_id}", + tags = ["system/silos"], + }] + async fn local_idp_user_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Set or invalidate user's password + /// + /// Passwords can only be updated for users in Silos with identity mode + /// `LocalOnly`. + #[endpoint { + method = POST, + path = "/v1/system/identity-providers/local/users/{user_id}/set-password", + tags = ["system/silos"], + }] + async fn local_idp_user_set_password( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + update: TypedBody, + ) -> Result; + + /// List projects + #[endpoint { + method = GET, + path = "/v1/projects", + tags = ["projects"], + }] + async fn project_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Create project + #[endpoint { + method = POST, + path = "/v1/projects", + tags = ["projects"], + }] + async fn project_create( + rqctx: RequestContext, + new_project: TypedBody, + ) -> Result, HttpError>; + + /// Fetch project + #[endpoint { + method = GET, + path = "/v1/projects/{project}", + tags = ["projects"], + }] + async fn project_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Delete project + #[endpoint { + method = DELETE, + path = "/v1/projects/{project}", + tags = ["projects"], + }] + async fn project_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + // TODO-correctness: Is it valid for PUT to accept application/json that's + // a subset of what the resource actually represents? If not, is that a + // problem? (HTTP may require that this be idempotent.) If so, can we get + // around that having this be a slightly different content-type (e.g., + // "application/json-patch")? We should see what other APIs do. + /// Update a project + #[endpoint { + method = PUT, + path = "/v1/projects/{project}", + tags = ["projects"], + }] + async fn project_update( + rqctx: RequestContext, + path_params: Path, + updated_project: TypedBody, + ) -> Result, HttpError>; + + /// Fetch project's IAM policy + #[endpoint { + method = GET, + path = "/v1/projects/{project}/policy", + tags = ["projects"], + }] + async fn project_policy_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError>; + + /// Update project's IAM policy + #[endpoint { + method = PUT, + path = "/v1/projects/{project}/policy", + tags = ["projects"], + }] + async fn project_policy_update( + rqctx: RequestContext, + path_params: Path, + new_policy: TypedBody>, + ) -> Result>, HttpError>; + + // IP Pools + + /// List IP pools + #[endpoint { + method = GET, + path = "/v1/ip-pools", + tags = ["projects"], + }] + async fn project_ip_pool_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Fetch IP pool + #[endpoint { + method = GET, + path = "/v1/ip-pools/{pool}", + tags = ["projects"], + }] + async fn project_ip_pool_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// List IP pools + #[endpoint { + method = GET, + path = "/v1/system/ip-pools", + tags = ["system/networking"], + }] + async fn ip_pool_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Create IP pool + #[endpoint { + method = POST, + path = "/v1/system/ip-pools", + tags = ["system/networking"], + }] + async fn ip_pool_create( + rqctx: RequestContext, + pool_params: TypedBody, + ) -> Result, HttpError>; + + /// Fetch IP pool + #[endpoint { + method = GET, + path = "/v1/system/ip-pools/{pool}", + tags = ["system/networking"], + }] + async fn ip_pool_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Delete IP pool + #[endpoint { + method = DELETE, + path = "/v1/system/ip-pools/{pool}", + tags = ["system/networking"], + }] + async fn ip_pool_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + /// Update IP pool + #[endpoint { + method = PUT, + path = "/v1/system/ip-pools/{pool}", + tags = ["system/networking"], + }] + async fn ip_pool_update( + rqctx: RequestContext, + path_params: Path, + updates: TypedBody, + ) -> Result, HttpError>; + + /// Fetch IP pool utilization + #[endpoint { + method = GET, + path = "/v1/system/ip-pools/{pool}/utilization", + tags = ["system/networking"], + }] + async fn ip_pool_utilization_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// List IP pool's linked silos + #[endpoint { + method = GET, + path = "/v1/system/ip-pools/{pool}/silos", + tags = ["system/networking"], + }] + async fn ip_pool_silo_list( + rqctx: RequestContext, + path_params: Path, + // paginating by resource_id because they're unique per pool. most robust + // option would be to paginate by a composite key representing the (pool, + // resource_type, resource) + query_params: Query, + // TODO: this could just list views::Silo -- it's not like knowing silo_id + // and nothing else is particularly useful -- except we also want to say + // whether the pool is marked default on each silo. So one option would + // be to do the same as we did with SiloIpPool -- include is_default on + // whatever the thing is. Still... all we'd have to do to make this usable + // in both places would be to make it { ...IpPool, silo_id, silo_name, + // is_default } + ) -> Result>, HttpError>; + + /// Link IP pool to silo + /// + /// Users in linked silos can allocate external IPs from this pool for their + /// instances. A silo can have at most one default pool. IPs are allocated from + /// the default pool when users ask for one without specifying a pool. + #[endpoint { + method = POST, + path = "/v1/system/ip-pools/{pool}/silos", + tags = ["system/networking"], + }] + async fn ip_pool_silo_link( + rqctx: RequestContext, + path_params: Path, + resource_assoc: TypedBody, + ) -> Result, HttpError>; + + /// Unlink IP pool from silo + /// + /// Will fail if there are any outstanding IPs allocated in the silo. + #[endpoint { + method = DELETE, + path = "/v1/system/ip-pools/{pool}/silos/{silo}", + tags = ["system/networking"], + }] + async fn ip_pool_silo_unlink( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + /// Make IP pool default for silo + /// + /// When a user asks for an IP (e.g., at instance create time) without + /// specifying a pool, the IP comes from the default pool if a default is + /// configured. When a pool is made the default for a silo, any existing + /// default will remain linked to the silo, but will no longer be the + /// default. + #[endpoint { + method = PUT, + path = "/v1/system/ip-pools/{pool}/silos/{silo}", + tags = ["system/networking"], + }] + async fn ip_pool_silo_update( + rqctx: RequestContext, + path_params: Path, + update: TypedBody, + ) -> Result, HttpError>; + + /// Fetch Oxide service IP pool + #[endpoint { + method = GET, + path = "/v1/system/ip-pools-service", + tags = ["system/networking"], + }] + async fn ip_pool_service_view( + rqctx: RequestContext, + ) -> Result, HttpError>; + + /// List ranges for IP pool + /// + /// Ranges are ordered by their first address. + #[endpoint { + method = GET, + path = "/v1/system/ip-pools/{pool}/ranges", + tags = ["system/networking"], + }] + async fn ip_pool_range_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError>; + + /// Add range to IP pool + /// + /// IPv6 ranges are not allowed yet. + #[endpoint { + method = POST, + path = "/v1/system/ip-pools/{pool}/ranges/add", + tags = ["system/networking"], + }] + async fn ip_pool_range_add( + rqctx: RequestContext, + path_params: Path, + range_params: TypedBody, + ) -> Result, HttpError>; + + /// Remove range from IP pool + #[endpoint { + method = POST, + path = "/v1/system/ip-pools/{pool}/ranges/remove", + tags = ["system/networking"], + }] + async fn ip_pool_range_remove( + rqctx: RequestContext, + path_params: Path, + range_params: TypedBody, + ) -> Result; + + /// List IP ranges for the Oxide service pool + /// + /// Ranges are ordered by their first address. + #[endpoint { + method = GET, + path = "/v1/system/ip-pools-service/ranges", + tags = ["system/networking"], + }] + async fn ip_pool_service_range_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Add IP range to Oxide service pool + /// + /// IPv6 ranges are not allowed yet. + #[endpoint { + method = POST, + path = "/v1/system/ip-pools-service/ranges/add", + tags = ["system/networking"], + }] + async fn ip_pool_service_range_add( + rqctx: RequestContext, + range_params: TypedBody, + ) -> Result, HttpError>; + + /// Remove IP range from Oxide service pool + #[endpoint { + method = POST, + path = "/v1/system/ip-pools-service/ranges/remove", + tags = ["system/networking"], + }] + async fn ip_pool_service_range_remove( + rqctx: RequestContext, + range_params: TypedBody, + ) -> Result; + + // Floating IP Addresses + + /// List floating IPs + #[endpoint { + method = GET, + path = "/v1/floating-ips", + tags = ["floating-ips"], + }] + async fn floating_ip_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Create floating IP + #[endpoint { + method = POST, + path = "/v1/floating-ips", + tags = ["floating-ips"], + }] + async fn floating_ip_create( + rqctx: RequestContext, + query_params: Query, + floating_params: TypedBody, + ) -> Result, HttpError>; + + /// Update floating IP + #[endpoint { + method = PUT, + path = "/v1/floating-ips/{floating_ip}", + tags = ["floating-ips"], + }] + async fn floating_ip_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + updated_floating_ip: TypedBody, + ) -> Result, HttpError>; + + /// Delete floating IP + #[endpoint { + method = DELETE, + path = "/v1/floating-ips/{floating_ip}", + tags = ["floating-ips"], + }] + async fn floating_ip_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Fetch floating IP + #[endpoint { + method = GET, + path = "/v1/floating-ips/{floating_ip}", + tags = ["floating-ips"] + }] + async fn floating_ip_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Attach floating IP + /// + /// Attach floating IP to an instance or other resource. + #[endpoint { + method = POST, + path = "/v1/floating-ips/{floating_ip}/attach", + tags = ["floating-ips"], + }] + async fn floating_ip_attach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + target: TypedBody, + ) -> Result, HttpError>; + + /// Detach floating IP + /// + // Detach floating IP from instance or other resource. + #[endpoint { + method = POST, + path = "/v1/floating-ips/{floating_ip}/detach", + tags = ["floating-ips"], + }] + async fn floating_ip_detach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + // Disks + + /// List disks + #[endpoint { + method = GET, + path = "/v1/disks", + tags = ["disks"], + }] + async fn disk_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + // TODO-correctness See note about instance create. This should be async. + /// Create a disk + #[endpoint { + method = POST, + path = "/v1/disks", + tags = ["disks"] + }] + async fn disk_create( + rqctx: RequestContext, + query_params: Query, + new_disk: TypedBody, + ) -> Result, HttpError>; + + /// Fetch disk + #[endpoint { + method = GET, + path = "/v1/disks/{disk}", + tags = ["disks"] + }] + async fn disk_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Delete disk + #[endpoint { + method = DELETE, + path = "/v1/disks/{disk}", + tags = ["disks"], + }] + async fn disk_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Fetch disk metrics + #[endpoint { + method = GET, + path = "/v1/disks/{disk}/metrics/{metric}", + tags = ["disks"], + }] + async fn disk_metrics_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query< + PaginationParams, + >, + selector_params: Query, + ) -> Result< + HttpResponseOk>, + HttpError, + >; + + /// Start importing blocks into disk + /// + /// Start the process of importing blocks into a disk + #[endpoint { + method = POST, + path = "/v1/disks/{disk}/bulk-write-start", + tags = ["disks"], + }] + async fn disk_bulk_write_import_start( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Import blocks into disk + #[endpoint { + method = POST, + path = "/v1/disks/{disk}/bulk-write", + tags = ["disks"], + }] + async fn disk_bulk_write_import( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + import_params: TypedBody, + ) -> Result; + + /// Stop importing blocks into disk + /// + /// Stop the process of importing blocks into a disk + #[endpoint { + method = POST, + path = "/v1/disks/{disk}/bulk-write-stop", + tags = ["disks"], + }] + async fn disk_bulk_write_import_stop( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Confirm disk block import completion + #[endpoint { + method = POST, + path = "/v1/disks/{disk}/finalize", + tags = ["disks"], + }] + async fn disk_finalize_import( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + finalize_params: TypedBody, + ) -> Result; + + // Instances + + /// List instances + #[endpoint { + method = GET, + path = "/v1/instances", + tags = ["instances"], + }] + async fn instance_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Create instance + #[endpoint { + method = POST, + path = "/v1/instances", + tags = ["instances"], + }] + async fn instance_create( + rqctx: RequestContext, + query_params: Query, + new_instance: TypedBody, + ) -> Result, HttpError>; + + /// Fetch instance + #[endpoint { + method = GET, + path = "/v1/instances/{instance}", + tags = ["instances"], + }] + async fn instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Delete instance + #[endpoint { + method = DELETE, + path = "/v1/instances/{instance}", + tags = ["instances"], + }] + async fn instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + /// Reboot an instance + #[endpoint { + method = POST, + path = "/v1/instances/{instance}/reboot", + tags = ["instances"], + }] + async fn instance_reboot( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Boot instance + #[endpoint { + method = POST, + path = "/v1/instances/{instance}/start", + tags = ["instances"], + }] + async fn instance_start( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Stop instance + #[endpoint { + method = POST, + path = "/v1/instances/{instance}/stop", + tags = ["instances"], + }] + async fn instance_stop( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError>; + + /// Fetch instance serial console + #[endpoint { + method = GET, + path = "/v1/instances/{instance}/serial-console", + tags = ["instances"], + }] + async fn instance_serial_console( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Stream instance serial console + #[channel { + protocol = WEBSOCKETS, + path = "/v1/instances/{instance}/serial-console/stream", + tags = ["instances"], + }] + async fn instance_serial_console_stream( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + conn: WebsocketConnection, + ) -> WebsocketChannelResult; + + /// List SSH public keys for instance + /// + /// List SSH public keys injected via cloud-init during instance creation. + /// Note that this list is a snapshot in time and will not reflect updates + /// made after the instance is created. + #[endpoint { + method = GET, + path = "/v1/instances/{instance}/ssh-public-keys", + tags = ["instances"], + }] + async fn instance_ssh_public_key_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result>, HttpError>; + + /// List disks for instance + #[endpoint { + method = GET, + path = "/v1/instances/{instance}/disks", + tags = ["instances"], + }] + async fn instance_disk_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + path_params: Path, + ) -> Result>, HttpError>; + + /// Attach disk to instance + #[endpoint { + method = POST, + path = "/v1/instances/{instance}/disks/attach", + tags = ["instances"], + }] + async fn instance_disk_attach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + disk_to_attach: TypedBody, + ) -> Result, HttpError>; + + /// Detach disk from instance + #[endpoint { + method = POST, + path = "/v1/instances/{instance}/disks/detach", + tags = ["instances"], + }] + async fn instance_disk_detach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + disk_to_detach: TypedBody, + ) -> Result, HttpError>; + + // Certificates + + /// List certificates for external endpoints + /// + /// Returns a list of TLS certificates used for the external API (for the + /// current Silo). These are sorted by creation date, with the most recent + /// certificates appearing first. + #[endpoint { + method = GET, + path = "/v1/certificates", + tags = ["silos"], + }] + async fn certificate_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Create new system-wide x.509 certificate + /// + /// This certificate is automatically used by the Oxide Control plane to serve + /// external connections. + #[endpoint { + method = POST, + path = "/v1/certificates", + tags = ["silos"] + }] + async fn certificate_create( + rqctx: RequestContext, + new_cert: TypedBody, + ) -> Result, HttpError>; + + /// Fetch certificate + /// + /// Returns the details of a specific certificate + #[endpoint { + method = GET, + path = "/v1/certificates/{certificate}", + tags = ["silos"], + }] + async fn certificate_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Delete certificate + /// + /// Permanently delete a certificate. This operation cannot be undone. + #[endpoint { + method = DELETE, + path = "/v1/certificates/{certificate}", + tags = ["silos"], + }] + async fn certificate_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + /// Create address lot + #[endpoint { + method = POST, + path = "/v1/system/networking/address-lot", + tags = ["system/networking"], + }] + async fn networking_address_lot_create( + rqctx: RequestContext, + new_address_lot: TypedBody, + ) -> Result, HttpError>; + + /// Delete address lot + #[endpoint { + method = DELETE, + path = "/v1/system/networking/address-lot/{address_lot}", + tags = ["system/networking"], + }] + async fn networking_address_lot_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + /// List address lots + #[endpoint { + method = GET, + path = "/v1/system/networking/address-lot", + tags = ["system/networking"], + }] + async fn networking_address_lot_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// List blocks in address lot + #[endpoint { + method = GET, + path = "/v1/system/networking/address-lot/{address_lot}/blocks", + tags = ["system/networking"], + }] + async fn networking_address_lot_block_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError>; + + /// Create loopback address + #[endpoint { + method = POST, + path = "/v1/system/networking/loopback-address", + tags = ["system/networking"], + }] + async fn networking_loopback_address_create( + rqctx: RequestContext, + new_loopback_address: TypedBody, + ) -> Result, HttpError>; + + /// Delete loopback address + #[endpoint { + method = DELETE, + path = "/v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask}", + tags = ["system/networking"], + }] + async fn networking_loopback_address_delete( + rqctx: RequestContext, + path: Path, + ) -> Result; + + /// List loopback addresses + #[endpoint { + method = GET, + path = "/v1/system/networking/loopback-address", + tags = ["system/networking"], + }] + async fn networking_loopback_address_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Create switch port settings + #[endpoint { + method = POST, + path = "/v1/system/networking/switch-port-settings", + tags = ["system/networking"], + }] + async fn networking_switch_port_settings_create( + rqctx: RequestContext, + new_settings: TypedBody, + ) -> Result, HttpError>; + + /// Delete switch port settings + #[endpoint { + method = DELETE, + path = "/v1/system/networking/switch-port-settings", + tags = ["system/networking"], + }] + async fn networking_switch_port_settings_delete( + rqctx: RequestContext, + query_params: Query, + ) -> Result; + + /// List switch port settings + #[endpoint { + method = GET, + path = "/v1/system/networking/switch-port-settings", + tags = ["system/networking"], + }] + async fn networking_switch_port_settings_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result>, HttpError>; + + /// Get information about switch port + #[endpoint { + method = GET, + path = "/v1/system/networking/switch-port-settings/{port}", + tags = ["system/networking"], + }] + async fn networking_switch_port_settings_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// List switch ports + #[endpoint { + method = GET, + path = "/v1/system/hardware/switch-port", + tags = ["system/hardware"], + }] + async fn networking_switch_port_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Get switch port status + #[endpoint { + method = GET, + path = "/v1/system/hardware/switch-port/{port}/status", + tags = ["system/hardware"], + }] + async fn networking_switch_port_status( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Apply switch port settings + #[endpoint { + method = POST, + path = "/v1/system/hardware/switch-port/{port}/settings", + tags = ["system/hardware"], + }] + async fn networking_switch_port_apply_settings( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + settings_body: TypedBody, + ) -> Result; + + /// Clear switch port settings + #[endpoint { + method = DELETE, + path = "/v1/system/hardware/switch-port/{port}/settings", + tags = ["system/hardware"], + }] + async fn networking_switch_port_clear_settings( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Create new BGP configuration + #[endpoint { + method = POST, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], + }] + async fn networking_bgp_config_create( + rqctx: RequestContext, + config: TypedBody, + ) -> Result, HttpError>; + + /// List BGP configurations + #[endpoint { + method = GET, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], + }] + async fn networking_bgp_config_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + //TODO pagination? the normal by-name/by-id stuff does not work here + /// Get BGP peer status + #[endpoint { + method = GET, + path = "/v1/system/networking/bgp-status", + tags = ["system/networking"], + }] + async fn networking_bgp_status( + rqctx: RequestContext, + ) -> Result>, HttpError>; + + //TODO pagination? the normal by-name/by-id stuff does not work here + /// Get BGP exported routes + #[endpoint { + method = GET, + path = "/v1/system/networking/bgp-exported", + tags = ["system/networking"], + }] + async fn networking_bgp_exported( + rqctx: RequestContext, + ) -> Result, HttpError>; + + /// Get BGP router message history + #[endpoint { + method = GET, + path = "/v1/system/networking/bgp-message-history", + tags = ["system/networking"], + }] + async fn networking_bgp_message_history( + rqctx: RequestContext, + query_params: Query, + ) -> Result, HttpError>; + + //TODO pagination? the normal by-name/by-id stuff does not work here + /// Get imported IPv4 BGP routes + #[endpoint { + method = GET, + path = "/v1/system/networking/bgp-routes-ipv4", + tags = ["system/networking"], + }] + async fn networking_bgp_imported_routes_ipv4( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Delete BGP configuration + #[endpoint { + method = DELETE, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], + }] + async fn networking_bgp_config_delete( + rqctx: RequestContext, + sel: Query, + ) -> Result; + + /// Update BGP announce set + /// + /// If the announce set exists, this endpoint replaces the existing announce + /// set with the one specified. + #[endpoint { + method = PUT, + path = "/v1/system/networking/bgp-announce-set", + tags = ["system/networking"], + }] + async fn networking_bgp_announce_set_update( + rqctx: RequestContext, + config: TypedBody, + ) -> Result, HttpError>; + + /// List BGP announce sets + #[endpoint { + method = GET, + path = "/v1/system/networking/bgp-announce-set", + tags = ["system/networking"], + }] + async fn networking_bgp_announce_set_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result>, HttpError>; + + /// Delete BGP announce set + #[endpoint { + method = DELETE, + path = "/v1/system/networking/bgp-announce-set/{name_or_id}", + tags = ["system/networking"], + }] + async fn networking_bgp_announce_set_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + // TODO: is pagination necessary here? How large do we expect the list of + // announcements to become in real usage? + /// Get originated routes for a specified BGP announce set + #[endpoint { + method = GET, + path = "/v1/system/networking/bgp-announce-set/{name_or_id}/announcement", + tags = ["system/networking"], + }] + async fn networking_bgp_announcement_list( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError>; + + /// Enable a BFD session + #[endpoint { + method = POST, + path = "/v1/system/networking/bfd-enable", + tags = ["system/networking"], + }] + async fn networking_bfd_enable( + rqctx: RequestContext, + session: TypedBody, + ) -> Result; + + /// Disable a BFD session + #[endpoint { + method = POST, + path = "/v1/system/networking/bfd-disable", + tags = ["system/networking"], + }] + async fn networking_bfd_disable( + rqctx: RequestContext, + session: TypedBody, + ) -> Result; + + /// Get BFD status + #[endpoint { + method = GET, + path = "/v1/system/networking/bfd-status", + tags = ["system/networking"], + }] + async fn networking_bfd_status( + rqctx: RequestContext, + ) -> Result>, HttpError>; + + /// Get user-facing services IP allowlist + #[endpoint { + method = GET, + path = "/v1/system/networking/allow-list", + tags = ["system/networking"], + }] + async fn networking_allow_list_view( + rqctx: RequestContext, + ) -> Result, HttpError>; + + /// Update user-facing services IP allowlist + #[endpoint { + method = PUT, + path = "/v1/system/networking/allow-list", + tags = ["system/networking"], + }] + async fn networking_allow_list_update( + rqctx: RequestContext, + params: TypedBody, + ) -> Result, HttpError>; + + // Images + + /// List images + /// + /// List images which are global or scoped to the specified project. The images + /// are returned sorted by creation date, with the most recent images appearing first. + #[endpoint { + method = GET, + path = "/v1/images", + tags = ["images"], + }] + async fn image_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result>, HttpError>; + + /// Create image + /// + /// Create a new image in a project. + #[endpoint { + method = POST, + path = "/v1/images", + tags = ["images"] + }] + async fn image_create( + rqctx: RequestContext, + query_params: Query, + new_image: TypedBody, + ) -> Result, HttpError>; + + /// Fetch image + /// + /// Fetch the details for a specific image in a project. + #[endpoint { + method = GET, + path = "/v1/images/{image}", + tags = ["images"], + }] + async fn image_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Delete image + /// + /// Permanently delete an image from a project. This operation cannot be undone. + /// Any instances in the project using the image will continue to run, however + /// new instances can not be created with this image. + #[endpoint { + method = DELETE, + path = "/v1/images/{image}", + tags = ["images"], + }] + async fn image_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Promote project image + /// + /// Promote project image to be visible to all projects in the silo + #[endpoint { + method = POST, + path = "/v1/images/{image}/promote", + tags = ["images"] + }] + async fn image_promote( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Demote silo image + /// + /// Demote silo image to be visible only to a specified project + #[endpoint { + method = POST, + path = "/v1/images/{image}/demote", + tags = ["images"] + }] + async fn image_demote( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// List network interfaces + #[endpoint { + method = GET, + path = "/v1/network-interfaces", + tags = ["instances"], + }] + async fn instance_network_interface_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Create network interface + #[endpoint { + method = POST, + path = "/v1/network-interfaces", + tags = ["instances"], + }] + async fn instance_network_interface_create( + rqctx: RequestContext, + query_params: Query, + interface_params: TypedBody, + ) -> Result, HttpError>; + + /// Delete network interface + /// + /// Note that the primary interface for an instance cannot be deleted if there + /// are any secondary interfaces. A new primary interface must be designated + /// first. The primary interface can be deleted if there are no secondary + /// interfaces. + #[endpoint { + method = DELETE, + path = "/v1/network-interfaces/{interface}", + tags = ["instances"], + }] + async fn instance_network_interface_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Fetch network interface + #[endpoint { + method = GET, + path = "/v1/network-interfaces/{interface}", + tags = ["instances"], + }] + async fn instance_network_interface_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Update network interface + #[endpoint { + method = PUT, + path = "/v1/network-interfaces/{interface}", + tags = ["instances"], + }] + async fn instance_network_interface_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + updated_iface: TypedBody, + ) -> Result, HttpError>; + + // External IP addresses for instances + + /// List external IP addresses + #[endpoint { + method = GET, + path = "/v1/instances/{instance}/external-ips", + tags = ["instances"], + }] + async fn instance_external_ip_list( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result>, HttpError>; + + /// Allocate and attach ephemeral IP to instance + #[endpoint { + method = POST, + path = "/v1/instances/{instance}/external-ips/ephemeral", + tags = ["instances"], + }] + async fn instance_ephemeral_ip_attach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ip_to_create: TypedBody, + ) -> Result, HttpError>; + + /// Detach and deallocate ephemeral IP from instance + #[endpoint { + method = DELETE, + path = "/v1/instances/{instance}/external-ips/ephemeral", + tags = ["instances"], + }] + async fn instance_ephemeral_ip_detach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + // Snapshots + + /// List snapshots + #[endpoint { + method = GET, + path = "/v1/snapshots", + tags = ["snapshots"], + }] + async fn snapshot_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Create snapshot + /// + /// Creates a point-in-time snapshot from a disk. + #[endpoint { + method = POST, + path = "/v1/snapshots", + tags = ["snapshots"], + }] + async fn snapshot_create( + rqctx: RequestContext, + query_params: Query, + new_snapshot: TypedBody, + ) -> Result, HttpError>; + + /// Fetch snapshot + #[endpoint { + method = GET, + path = "/v1/snapshots/{snapshot}", + tags = ["snapshots"], + }] + async fn snapshot_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Delete snapshot + #[endpoint { + method = DELETE, + path = "/v1/snapshots/{snapshot}", + tags = ["snapshots"], + }] + async fn snapshot_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + // VPCs + + /// List VPCs + #[endpoint { + method = GET, + path = "/v1/vpcs", + tags = ["vpcs"], + }] + async fn vpc_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Create VPC + #[endpoint { + method = POST, + path = "/v1/vpcs", + tags = ["vpcs"], + }] + async fn vpc_create( + rqctx: RequestContext, + query_params: Query, + body: TypedBody, + ) -> Result, HttpError>; + + /// Fetch VPC + #[endpoint { + method = GET, + path = "/v1/vpcs/{vpc}", + tags = ["vpcs"], + }] + async fn vpc_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Update a VPC + #[endpoint { + method = PUT, + path = "/v1/vpcs/{vpc}", + tags = ["vpcs"], + }] + async fn vpc_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + updated_vpc: TypedBody, + ) -> Result, HttpError>; + + /// Delete VPC + #[endpoint { + method = DELETE, + path = "/v1/vpcs/{vpc}", + tags = ["vpcs"], + }] + async fn vpc_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// List subnets + #[endpoint { + method = GET, + path = "/v1/vpc-subnets", + tags = ["vpcs"], + }] + async fn vpc_subnet_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Create subnet + #[endpoint { + method = POST, + path = "/v1/vpc-subnets", + tags = ["vpcs"], + }] + async fn vpc_subnet_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError>; + + /// Fetch subnet + #[endpoint { + method = GET, + path = "/v1/vpc-subnets/{subnet}", + tags = ["vpcs"], + }] + async fn vpc_subnet_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Delete subnet + #[endpoint { + method = DELETE, + path = "/v1/vpc-subnets/{subnet}", + tags = ["vpcs"], + }] + async fn vpc_subnet_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Update subnet + #[endpoint { + method = PUT, + path = "/v1/vpc-subnets/{subnet}", + tags = ["vpcs"], + }] + async fn vpc_subnet_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + subnet_params: TypedBody, + ) -> Result, HttpError>; + + // This endpoint is likely temporary. We would rather list all IPs allocated in + // a subnet whether they come from NICs or something else. See + // https://github.com/oxidecomputer/omicron/issues/2476 + + /// List network interfaces + #[endpoint { + method = GET, + path = "/v1/vpc-subnets/{subnet}/network-interfaces", + tags = ["vpcs"], + }] + async fn vpc_subnet_list_network_interfaces( + rqctx: RequestContext, + path_params: Path, + query_params: Query>, + ) -> Result>, HttpError>; + + // VPC Firewalls + + /// List firewall rules + #[endpoint { + method = GET, + path = "/v1/vpc-firewall-rules", + tags = ["vpcs"], + }] + async fn vpc_firewall_rules_view( + rqctx: RequestContext, + query_params: Query, + ) -> Result, HttpError>; + + // Note: the limits in the below comment come from the firewall rules model + // file, nexus/db-model/src/vpc_firewall_rule.rs. + + /// Replace firewall rules + /// + /// The maximum number of rules per VPC is 1024. + /// + /// Targets are used to specify the set of instances to which a firewall rule + /// applies. You can target instances directly by name, or specify a VPC, VPC + /// subnet, IP, or IP subnet, which will apply the rule to traffic going to + /// all matching instances. Targets are additive: the rule applies to instances + /// matching ANY target. The maximum number of targets is 256. + /// + /// Filters reduce the scope of a firewall rule. Without filters, the rule + /// applies to all packets to the targets (or from the targets, if it's an + /// outbound rule). With multiple filters, the rule applies only to packets + /// matching ALL filters. The maximum number of each type of filter is 256. + #[endpoint { + method = PUT, + path = "/v1/vpc-firewall-rules", + tags = ["vpcs"], + }] + async fn vpc_firewall_rules_update( + rqctx: RequestContext, + query_params: Query, + router_params: TypedBody, + ) -> Result, HttpError>; + + // VPC Routers + + /// List routers + #[endpoint { + method = GET, + path = "/v1/vpc-routers", + tags = ["vpcs"], + }] + async fn vpc_router_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Fetch router + #[endpoint { + method = GET, + path = "/v1/vpc-routers/{router}", + tags = ["vpcs"], + }] + async fn vpc_router_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Create VPC router + #[endpoint { + method = POST, + path = "/v1/vpc-routers", + tags = ["vpcs"], + }] + async fn vpc_router_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError>; + + /// Delete router + #[endpoint { + method = DELETE, + path = "/v1/vpc-routers/{router}", + tags = ["vpcs"], + }] + async fn vpc_router_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Update router + #[endpoint { + method = PUT, + path = "/v1/vpc-routers/{router}", + tags = ["vpcs"], + }] + async fn vpc_router_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + router_params: TypedBody, + ) -> Result, HttpError>; + + /// List routes + /// + /// List the routes associated with a router in a particular VPC. + #[endpoint { + method = GET, + path = "/v1/vpc-router-routes", + tags = ["vpcs"], + }] + async fn vpc_router_route_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + // Vpc Router Routes + + /// Fetch route + #[endpoint { + method = GET, + path = "/v1/vpc-router-routes/{route}", + tags = ["vpcs"], + }] + async fn vpc_router_route_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Create route + #[endpoint { + method = POST, + path = "/v1/vpc-router-routes", + tags = ["vpcs"], + }] + async fn vpc_router_route_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError>; + + /// Delete route + #[endpoint { + method = DELETE, + path = "/v1/vpc-router-routes/{route}", + tags = ["vpcs"], + }] + async fn vpc_router_route_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Update route + #[endpoint { + method = PUT, + path = "/v1/vpc-router-routes/{route}", + tags = ["vpcs"], + }] + async fn vpc_router_route_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + router_params: TypedBody, + ) -> Result, HttpError>; + + // Racks + + /// List racks + #[endpoint { + method = GET, + path = "/v1/system/hardware/racks", + tags = ["system/hardware"], + }] + async fn rack_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Fetch rack + #[endpoint { + method = GET, + path = "/v1/system/hardware/racks/{rack_id}", + tags = ["system/hardware"], + }] + async fn rack_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// List uninitialized sleds + #[endpoint { + method = GET, + path = "/v1/system/hardware/sleds-uninitialized", + tags = ["system/hardware"] + }] + async fn sled_list_uninitialized( + rqctx: RequestContext, + query: Query>, + ) -> Result>, HttpError>; + + /// Add sled to initialized rack + // + // TODO: In the future this should really be a PUT request, once we resolve + // https://github.com/oxidecomputer/omicron/issues/4494. It should also + // explicitly be tied to a rack via a `rack_id` path param. For now we assume + // we are only operating on single rack systems. + #[endpoint { + method = POST, + path = "/v1/system/hardware/sleds", + tags = ["system/hardware"] + }] + async fn sled_add( + rqctx: RequestContext, + sled: TypedBody, + ) -> Result, HttpError>; + + // Sleds + + /// List sleds + #[endpoint { + method = GET, + path = "/v1/system/hardware/sleds", + tags = ["system/hardware"], + }] + async fn sled_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Fetch sled + #[endpoint { + method = GET, + path = "/v1/system/hardware/sleds/{sled_id}", + tags = ["system/hardware"], + }] + async fn sled_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Set sled provision policy + #[endpoint { + method = PUT, + path = "/v1/system/hardware/sleds/{sled_id}/provision-policy", + tags = ["system/hardware"], + }] + async fn sled_set_provision_policy( + rqctx: RequestContext, + path_params: Path, + new_provision_state: TypedBody, + ) -> Result, HttpError>; + + /// List instances running on given sled + #[endpoint { + method = GET, + path = "/v1/system/hardware/sleds/{sled_id}/instances", + tags = ["system/hardware"], + }] + async fn sled_instance_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError>; + + // Physical disks + + /// List physical disks + #[endpoint { + method = GET, + path = "/v1/system/hardware/disks", + tags = ["system/hardware"], + }] + async fn physical_disk_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Get a physical disk + #[endpoint { + method = GET, + path = "/v1/system/hardware/disks/{disk_id}", + tags = ["system/hardware"], + }] + async fn physical_disk_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + // Switches + + /// List switches + #[endpoint { + method = GET, + path = "/v1/system/hardware/switches", + tags = ["system/hardware"], + }] + async fn switch_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Fetch switch + #[endpoint { + method = GET, + path = "/v1/system/hardware/switches/{switch_id}", + tags = ["system/hardware"], + }] + async fn switch_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// List physical disks attached to sleds + #[endpoint { + method = GET, + path = "/v1/system/hardware/sleds/{sled_id}/disks", + tags = ["system/hardware"], + }] + async fn sled_physical_disk_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError>; + + // Metrics + + /// View metrics + /// + /// View CPU, memory, or storage utilization metrics at the fleet or silo level. + #[endpoint { + method = GET, + path = "/v1/system/metrics/{metric_name}", + tags = ["system/metrics"], + }] + async fn system_metric( + rqctx: RequestContext, + path_params: Path, + pag_params: Query< + PaginationParams, + >, + other_params: Query, + ) -> Result< + HttpResponseOk>, + HttpError, + >; + + /// View metrics + /// + /// View CPU, memory, or storage utilization metrics at the silo or project level. + #[endpoint { + method = GET, + path = "/v1/metrics/{metric_name}", + tags = ["metrics"], + }] + async fn silo_metric( + rqctx: RequestContext, + path_params: Path, + pag_params: Query< + PaginationParams, + >, + other_params: Query, + ) -> Result< + HttpResponseOk>, + HttpError, + >; + + /// List timeseries schemas + #[endpoint { + method = GET, + path = "/v1/timeseries/schema", + tags = ["metrics"], + }] + async fn timeseries_schema_list( + rqctx: RequestContext, + pag_params: Query, + ) -> Result< + HttpResponseOk>, + HttpError, + >; + + // TODO: can we link to an OxQL reference? Do we have one? Can we even do links? + + /// Run timeseries query + /// + /// Queries are written in OxQL. + #[endpoint { + method = POST, + path = "/v1/timeseries/query", + tags = ["metrics"], + }] + async fn timeseries_query( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError>; + + // Updates + + /// Upload TUF repository + #[endpoint { + method = PUT, + path = "/v1/system/update/repository", + tags = ["system/update"], + unpublished = true, + }] + async fn system_update_put_repository( + rqctx: RequestContext, + query: Query, + body: StreamingBody, + ) -> Result, HttpError>; + + /// Fetch TUF repository description + /// + /// Fetch description of TUF repository by system version. + #[endpoint { + method = GET, + path = "/v1/system/update/repository/{system_version}", + tags = ["system/update"], + unpublished = true, + }] + async fn system_update_get_repository( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + // Silo users + + /// List users + #[endpoint { + method = GET, + path = "/v1/users", + tags = ["silos"], + }] + async fn user_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + // Silo groups + + /// List groups + #[endpoint { + method = GET, + path = "/v1/groups", + tags = ["silos"], + }] + async fn group_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Fetch group + #[endpoint { + method = GET, + path = "/v1/groups/{group_id}", + tags = ["silos"], + }] + async fn group_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + // Built-in (system) users + + /// List built-in users + #[endpoint { + method = GET, + path = "/v1/system/users-builtin", + tags = ["system/silos"], + }] + async fn user_builtin_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Fetch built-in user + #[endpoint { + method = GET, + path = "/v1/system/users-builtin/{user}", + tags = ["system/silos"], + }] + async fn user_builtin_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + // Built-in roles + + /// List built-in roles + #[endpoint { + method = GET, + path = "/v1/system/roles", + tags = ["roles"], + }] + async fn role_list( + rqctx: RequestContext, + query_params: Query< + PaginationParams, + >, + ) -> Result>, HttpError>; + + /// Fetch built-in role + #[endpoint { + method = GET, + path = "/v1/system/roles/{role_name}", + tags = ["roles"], + }] + async fn role_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + // Current user + + /// Fetch user for current session + #[endpoint { + method = GET, + path = "/v1/me", + tags = ["session"], + }] + async fn current_user_view( + rqctx: RequestContext, + ) -> Result, HttpError>; + + /// Fetch current user's groups + #[endpoint { + method = GET, + path = "/v1/me/groups", + tags = ["session"], + }] + async fn current_user_groups( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + // Per-user SSH public keys + + /// List SSH public keys + /// + /// Lists SSH public keys for the currently authenticated user. + #[endpoint { + method = GET, + path = "/v1/me/ssh-keys", + tags = ["session"], + }] + async fn current_user_ssh_key_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError>; + + /// Create SSH public key + /// + /// Create an SSH public key for the currently authenticated user. + #[endpoint { + method = POST, + path = "/v1/me/ssh-keys", + tags = ["session"], + }] + async fn current_user_ssh_key_create( + rqctx: RequestContext, + new_key: TypedBody, + ) -> Result, HttpError>; + + /// Fetch SSH public key + /// + /// Fetch SSH public key associated with the currently authenticated user. + #[endpoint { + method = GET, + path = "/v1/me/ssh-keys/{ssh_key}", + tags = ["session"], + }] + async fn current_user_ssh_key_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Delete SSH public key + /// + /// Delete an SSH public key associated with the currently authenticated user. + #[endpoint { + method = DELETE, + path = "/v1/me/ssh-keys/{ssh_key}", + tags = ["session"], + }] + async fn current_user_ssh_key_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result; + + // Probes (experimental) + + /// List instrumentation probes + #[endpoint { + method = GET, + path = "/experimental/v1/probes", + tags = ["hidden"], // system/probes: only one tag is allowed + }] + async fn probe_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// View instrumentation probe + #[endpoint { + method = GET, + path = "/experimental/v1/probes/{probe}", + tags = ["hidden"], // system/probes: only one tag is allowed + }] + async fn probe_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Create instrumentation probe + #[endpoint { + method = POST, + path = "/experimental/v1/probes", + tags = ["hidden"], // system/probes: only one tag is allowed + }] + async fn probe_create( + rqctx: RequestContext, + query_params: Query, + new_probe: TypedBody, + ) -> Result, HttpError>; + + /// Delete instrumentation probe + #[endpoint { + method = DELETE, + path = "/experimental/v1/probes/{probe}", + tags = ["hidden"], // system/probes: only one tag is allowed + }] + async fn probe_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result; + + // Console API: logins + + /// SAML login console page (just a link to the IdP) + #[endpoint { + method = GET, + path = "/login/{silo_name}/saml/{provider_name}", + tags = ["login"], + unpublished = true, + }] + async fn login_saml_begin( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Get a redirect straight to the IdP + /// + /// Console uses this to avoid having to ask the API anything about the IdP. It + /// already knows the IdP name from the path, so it can just link to this path + /// and rely on Nexus to redirect to the actual IdP. + #[endpoint { + method = GET, + path = "/login/{silo_name}/saml/{provider_name}/redirect", + tags = ["login"], + unpublished = true, + }] + async fn login_saml_redirect( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// Authenticate a user via SAML + #[endpoint { + method = POST, + path = "/login/{silo_name}/saml/{provider_name}", + tags = ["login"], + }] + async fn login_saml( + rqctx: RequestContext, + path_params: Path, + body_bytes: dropshot::UntypedBody, + ) -> Result; + + #[endpoint { + method = GET, + path = "/login/{silo_name}/local", + tags = ["login"], + unpublished = true, + }] + async fn login_local_begin( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Authenticate a user via username and password + #[endpoint { + method = POST, + path = "/v1/login/{silo_name}/local", + tags = ["login"], + }] + async fn login_local( + rqctx: RequestContext, + path_params: Path, + credentials: TypedBody, + ) -> Result, HttpError>; + + /// Log user out of web console by deleting session on client and server + #[endpoint { + // important for security that this be a POST despite the empty req body + method = POST, + path = "/v1/logout", + tags = ["hidden"], + }] + async fn logout( + rqctx: RequestContext, + cookies: Cookies, + ) -> Result, HttpError>; + + /// Redirect to a login page for the current Silo (if that can be determined) + #[endpoint { + method = GET, + path = "/login", + unpublished = true, + }] + async fn login_begin( + rqctx: RequestContext, + query_params: Query, + ) -> Result; + + // Console API: Pages + + #[endpoint { + method = GET, + path = "/projects/{path:.*}", + unpublished = true, + }] + async fn console_projects( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + #[endpoint { + method = GET, + path = "/settings/{path:.*}", + unpublished = true, + }] + async fn console_settings_page( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + #[endpoint { + method = GET, + path = "/system/{path:.*}", + unpublished = true, + }] + async fn console_system_page( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + #[endpoint { + method = GET, + path = "/lookup/{path:.*}", + unpublished = true, + }] + async fn console_lookup( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + #[endpoint { + method = GET, + path = "/", + unpublished = true, + }] + async fn console_root( + rqctx: RequestContext, + ) -> Result, HttpError>; + + #[endpoint { + method = GET, + path = "/projects-new", + unpublished = true, + }] + async fn console_projects_new( + rqctx: RequestContext, + ) -> Result, HttpError>; + + #[endpoint { + method = GET, + path = "/images", + unpublished = true, + }] + async fn console_silo_images( + rqctx: RequestContext, + ) -> Result, HttpError>; + + #[endpoint { + method = GET, + path = "/utilization", + unpublished = true, + }] + async fn console_silo_utilization( + rqctx: RequestContext, + ) -> Result, HttpError>; + + #[endpoint { + method = GET, + path = "/access", + unpublished = true, + }] + async fn console_silo_access( + rqctx: RequestContext, + ) -> Result, HttpError>; + + /// Serve a static asset + #[endpoint { + method = GET, + path = "/assets/{path:.*}", + unpublished = true, + }] + async fn asset( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError>; + + /// Start an OAuth 2.0 Device Authorization Grant + /// + /// This endpoint is designed to be accessed from an *unauthenticated* + /// API client. It generates and records a `device_code` and `user_code` + /// which must be verified and confirmed prior to a token being granted. + #[endpoint { + method = POST, + path = "/device/auth", + content_type = "application/x-www-form-urlencoded", + tags = ["hidden"], // "token" + }] + async fn device_auth_request( + rqctx: RequestContext, + params: TypedBody, + ) -> Result, HttpError>; + + /// Verify an OAuth 2.0 Device Authorization Grant + /// + /// This endpoint should be accessed in a full user agent (e.g., + /// a browser). If the user is not logged in, we redirect them to + /// the login page and use the `state` parameter to get them back + /// here on completion. If they are logged in, serve up the console + /// verification page so they can verify the user code. + #[endpoint { + method = GET, + path = "/device/verify", + unpublished = true, + }] + async fn device_auth_verify( + rqctx: RequestContext, + ) -> Result, HttpError>; + + #[endpoint { + method = GET, + path = "/device/success", + unpublished = true, + }] + async fn device_auth_success( + rqctx: RequestContext, + ) -> Result, HttpError>; + + /// Confirm an OAuth 2.0 Device Authorization Grant + /// + /// This endpoint is designed to be accessed by the user agent (browser), + /// not the client requesting the token. So we do not actually return the + /// token here; it will be returned in response to the poll on `/device/token`. + #[endpoint { + method = POST, + path = "/device/confirm", + tags = ["hidden"], // "token" + }] + async fn device_auth_confirm( + rqctx: RequestContext, + params: TypedBody, + ) -> Result; + + /// Request a device access token + /// + /// This endpoint should be polled by the client until the user code + /// is verified and the grant is confirmed. + #[endpoint { + method = POST, + path = "/device/token", + content_type = "application/x-www-form-urlencoded", + tags = ["hidden"], // "token" + }] + async fn device_access_token( + rqctx: RequestContext, + params: TypedBody, + ) -> Result, HttpError>; +} + +/// Perform extra validations on the OpenAPI spec. +pub fn validate_api(spec: &OpenAPI, mut cx: ValidationContext<'_>) { + if spec.openapi != "3.0.3" { + cx.report_error(anyhow!( + "Expected OpenAPI version to be 3.0.3, found {}", + spec.openapi, + )); + } + if spec.info.title != "Oxide Region API" { + cx.report_error(anyhow!( + "Expected OpenAPI version to be 'Oxide Region API', found '{}'", + spec.info.title, + )); + } + if spec.info.version != API_VERSION { + cx.report_error(anyhow!( + "Expected OpenAPI version to be '{}', found '{}'", + API_VERSION, + spec.info.version, + )); + } + + // Spot check a couple of items. + if spec.paths.paths.is_empty() { + cx.report_error(anyhow!("Expected at least one path in the spec")); + } + if spec.paths.paths.get("/v1/projects").is_none() { + cx.report_error(anyhow!("Expected a path for /v1/projects")); + } + + // Construct a string that helps us identify the organization of tags and + // operations. + let mut ops_by_tag = + BTreeMap::>::new(); + + let mut ops_by_tag_valid = true; + for (path, method, op) in spec.operations() { + // Make sure each operation has exactly one tag. Note, we intentionally + // do this before validating the OpenAPI output as fixing an error here + // would necessitate refreshing the spec file again. + if op.tags.len() != 1 { + cx.report_error(anyhow!( + "operation '{}' has {} tags rather than 1", + op.operation_id.as_ref().unwrap(), + op.tags.len() + )); + ops_by_tag_valid = false; + continue; + } + + // Every non-hidden endpoint must have a summary + if op.tags.contains(&"hidden".to_string()) && op.summary.is_none() { + cx.report_error(anyhow!( + "operation '{}' is missing a summary doc comment", + op.operation_id.as_ref().unwrap() + )); + // This error does not prevent `ops_by_tag` from being populated + // correctly, so we can continue. + } + + ops_by_tag + .entry(op.tags.first().unwrap().to_string()) + .or_default() + .push(( + op.operation_id.as_ref().unwrap().to_string(), + method.to_string().to_uppercase(), + path.to_string(), + )); + } + + if ops_by_tag_valid { + let mut tags = String::new(); + for (tag, mut ops) in ops_by_tag { + ops.sort(); + tags.push_str(&format!( + r#"API operations found with tag "{}""#, + tag + )); + tags.push_str(&format!( + "\n{:40} {:8} {}\n", + "OPERATION ID", "METHOD", "URL PATH" + )); + for (operation_id, method, path) in ops { + tags.push_str(&format!( + "{:40} {:8} {}\n", + operation_id, method, path + )); + } + tags.push('\n'); + } + + // When this fails, verify that operations on which you're adding, + // renaming, or changing the tags are what you intend. + cx.record_file_contents( + "nexus/external-api/output/nexus_tags.txt", + tags.into_bytes(), + ); + } +} + +pub type IpPoolRangePaginationParams = + PaginationParams; + +/// Type used to paginate request to list timeseries schema. +pub type TimeseriesSchemaPaginationParams = + PaginationParams; diff --git a/nexus/internal-api/src/lib.rs b/nexus/internal-api/src/lib.rs index 7ac3e42f57..12e99ba23b 100644 --- a/nexus/internal-api/src/lib.rs +++ b/nexus/internal-api/src/lib.rs @@ -33,14 +33,14 @@ use omicron_common::{ DiskRuntimeState, DownstairsClientStopRequest, DownstairsClientStopped, ProducerEndpoint, ProducerRegistrationResponse, RepairFinishInfo, RepairProgress, - RepairStartInfo, SledInstanceState, + RepairStartInfo, SledVmmState, }, }, update::ArtifactId, }; use omicron_uuid_kinds::{ - DemoSagaUuid, DownstairsKind, SledUuid, TypedUuid, UpstairsKind, - UpstairsRepairKind, + DemoSagaUuid, DownstairsKind, PropolisUuid, SledUuid, TypedUuid, + UpstairsKind, UpstairsRepairKind, }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -108,15 +108,15 @@ pub trait NexusInternalApi { body: TypedBody, ) -> Result, HttpError>; - /// Report updated state for an instance. + /// Report updated state for a VMM. #[endpoint { method = PUT, - path = "/instances/{instance_id}", + path = "/vmms/{propolis_id}", }] async fn cpapi_instances_put( rqctx: RequestContext, - path_params: Path, - new_runtime_state: TypedBody, + path_params: Path, + new_runtime_state: TypedBody, ) -> Result; #[endpoint { @@ -568,6 +568,12 @@ pub struct InstancePathParam { pub instance_id: Uuid, } +/// Path parameters for VMM requests (internal API) +#[derive(Deserialize, JsonSchema)] +pub struct VmmPathParam { + pub propolis_id: PropolisUuid, +} + #[derive(Clone, Copy, Debug, Deserialize, JsonSchema, Serialize)] pub struct CollectorIdPathParams { /// The ID of the oximeter collector. diff --git a/nexus/reconfigurator/execution/src/datasets.rs b/nexus/reconfigurator/execution/src/datasets.rs index 6444934ba6..2f84378a13 100644 --- a/nexus/reconfigurator/execution/src/datasets.rs +++ b/nexus/reconfigurator/execution/src/datasets.rs @@ -67,7 +67,7 @@ pub(crate) async fn ensure_dataset_records_exist( id.into_untyped_uuid(), pool_id.into_untyped_uuid(), Some(address), - kind.into(), + kind.clone(), ); let maybe_inserted = datastore .dataset_insert_if_not_exists(dataset) diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index 9ab84e15ff..aab3839bd0 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -515,6 +515,7 @@ mod test { use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::policy::BOUNDARY_NTP_REDUNDANCY; use omicron_common::policy::COCKROACHDB_REDUNDANCY; + use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_common::policy::NEXUS_REDUNDANCY; use omicron_common::zpool_name::ZpoolName; use omicron_test_utils::dev::test_setup_log; @@ -1526,6 +1527,7 @@ mod test { service_nic_rows: &[], target_boundary_ntp_zone_count: BOUNDARY_NTP_REDUNDANCY, target_nexus_zone_count: NEXUS_REDUNDANCY, + target_internal_dns_zone_count: INTERNAL_DNS_REDUNDANCY, target_cockroachdb_zone_count: COCKROACHDB_REDUNDANCY, target_cockroachdb_cluster_version: CockroachDbClusterVersion::POLICY, diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs index 019830442a..607c929a19 100644 --- a/nexus/reconfigurator/execution/src/lib.rs +++ b/nexus/reconfigurator/execution/src/lib.rs @@ -229,7 +229,6 @@ pub async fn realize_blueprint_with_overrides( ); // All steps are registered, so execute the engine. - let result = engine.execute().await?; Ok(output.into_value(result.token()).await) @@ -650,6 +649,7 @@ mod tests { use nexus_db_model::SledSystemHardware; use nexus_db_model::SledUpdate; use nexus_db_model::Zpool; + use omicron_common::api::external::Error; use std::collections::BTreeSet; use uuid::Uuid; @@ -720,10 +720,10 @@ mod tests { PhysicalDiskKind::U2, sled_id.into_untyped_uuid(), ); - datastore - .physical_disk_insert(&opctx, disk.clone()) - .await - .expect("failed to upsert physical disk"); + match datastore.physical_disk_insert(&opctx, disk.clone()).await { + Ok(_) | Err(Error::ObjectAlreadyExists { .. }) => (), + Err(e) => panic!("failed to upsert physical disk: {e}"), + } if pool_inserted.insert(pool_id) { let zpool = Zpool::new( diff --git a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs index af95eb8e77..d94bbe2e27 100644 --- a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs +++ b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs @@ -135,7 +135,6 @@ mod test { use httptest::responders::status_code; use httptest::Expectation; use nexus_db_model::Dataset; - use nexus_db_model::DatasetKind; use nexus_db_model::PhysicalDisk; use nexus_db_model::PhysicalDiskKind; use nexus_db_model::PhysicalDiskPolicy; @@ -153,6 +152,7 @@ mod test { use nexus_types::identity::Asset; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Generation; + use omicron_common::api::internal::shared::DatasetKind; use omicron_common::disk::DiskIdentity; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::PhysicalDiskUuid; diff --git a/nexus/reconfigurator/execution/src/test_utils.rs b/nexus/reconfigurator/execution/src/test_utils.rs index 8a1afac3ac..6af82ef9dd 100644 --- a/nexus/reconfigurator/execution/src/test_utils.rs +++ b/nexus/reconfigurator/execution/src/test_utils.rs @@ -38,9 +38,9 @@ pub(crate) async fn realize_blueprint_and_expect( sender, ) .await - // We expect here rather than in the caller because we want to assert - // that the result is a `RealizeBlueprintOutput` (which is `must_use` - // so it must be assigned to `_`). + // We expect here rather than in the caller because we want to assert that + // the result is a `RealizeBlueprintOutput`. Because the latter is + // `must_use`, the caller may assign it to `_` and miss the `expect` call. .expect("failed to execute blueprint"); let buffer = receiver_task.await.expect("failed to receive events"); diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 2d8a7c9598..c7eb5bddad 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -37,13 +37,17 @@ use nexus_types::external_api::views::SledState; use omicron_common::address::get_internal_dns_server_addresses; use omicron_common::address::get_sled_address; use omicron_common::address::get_switch_zone_address; +use omicron_common::address::ReservedRackSubnet; use omicron_common::address::CP_SERVICES_RESERVED_ADDRESSES; +use omicron_common::address::DNS_HTTP_PORT; +use omicron_common::address::DNS_PORT; use omicron_common::address::NTP_PORT; use omicron_common::address::SLED_RESERVED_ADDRESSES; use omicron_common::api::external::Generation; use omicron_common::api::external::Vni; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; +use omicron_common::policy::MAX_INTERNAL_DNS_REDUNDANCY; use omicron_uuid_kinds::ExternalIpKind; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneKind; @@ -73,6 +77,7 @@ use typed_rng::UuidRng; use super::external_networking::BuilderExternalNetworking; use super::external_networking::ExternalNetworkingChoice; use super::external_networking::ExternalSnatNetworkingChoice; +use super::internal_dns::DnsSubnetAllocator; use super::zones::is_already_expunged; use super::zones::BuilderZoneState; use super::zones::BuilderZonesConfig; @@ -106,6 +111,12 @@ pub enum Error { }, #[error("programming error in planner")] Planner(#[source] anyhow::Error), + #[error("no reserved subnets available for DNS")] + NoAvailableDnsSubnets, + #[error( + "can only have {MAX_INTERNAL_DNS_REDUNDANCY} internal DNS servers" + )] + TooManyDnsServers, } /// Describes whether an idempotent "ensure" operation resulted in action taken @@ -197,6 +208,7 @@ pub struct BlueprintBuilder<'a> { input: &'a PlanningInput, sled_ip_allocators: BTreeMap, external_networking: BuilderExternalNetworking<'a>, + internal_dns_subnets: DnsSubnetAllocator, // These fields will become part of the final blueprint. See the // corresponding fields in `Blueprint`. @@ -291,6 +303,8 @@ impl<'a> BlueprintBuilder<'a> { let external_networking = BuilderExternalNetworking::new(parent_blueprint, input)?; + let internal_dns_subnets = + DnsSubnetAllocator::new(parent_blueprint, input)?; // Prefer the sled state from our parent blueprint for sleds // that were in it; there may be new sleds in `input`, in which @@ -323,6 +337,7 @@ impl<'a> BlueprintBuilder<'a> { input, sled_ip_allocators: BTreeMap::new(), external_networking, + internal_dns_subnets, zones: BlueprintZonesBuilder::new(parent_blueprint), disks: BlueprintDisksBuilder::new(parent_blueprint), sled_state, @@ -619,6 +634,69 @@ impl<'a> BlueprintBuilder<'a> { Ok(EnsureMultiple::Changed { added, removed }) } + fn sled_add_zone_internal_dns( + &mut self, + sled_id: SledUuid, + gz_address_index: u32, + ) -> Result { + let sled_subnet = self.sled_resources(sled_id)?.subnet; + let rack_subnet = ReservedRackSubnet::from_subnet(sled_subnet); + let dns_subnet = self.internal_dns_subnets.alloc(rack_subnet)?; + let address = dns_subnet.dns_address(); + let zpool = self.sled_select_zpool(sled_id, ZoneKind::InternalDns)?; + let zone_type = + BlueprintZoneType::InternalDns(blueprint_zone_type::InternalDns { + dataset: OmicronZoneDataset { pool_name: zpool.clone() }, + dns_address: SocketAddrV6::new(address, DNS_PORT, 0, 0), + http_address: SocketAddrV6::new(address, DNS_HTTP_PORT, 0, 0), + gz_address: dns_subnet.gz_address(), + gz_address_index, + }); + + let zone = BlueprintZoneConfig { + disposition: BlueprintZoneDisposition::InService, + id: self.rng.zone_rng.next(), + underlay_address: address, + filesystem_pool: Some(zpool), + zone_type, + }; + + self.sled_add_zone(sled_id, zone)?; + Ok(Ensure::Added) + } + + pub fn sled_ensure_zone_multiple_internal_dns( + &mut self, + sled_id: SledUuid, + desired_zone_count: usize, + ) -> Result { + // How many internal DNS zones do we need to add? + let count = + self.sled_num_running_zones_of_kind(sled_id, ZoneKind::InternalDns); + let to_add = match desired_zone_count.checked_sub(count) { + Some(0) => return Ok(EnsureMultiple::NotNeeded), + Some(n) => n, + None => { + return Err(Error::Planner(anyhow!( + "removing an internal DNS zone not yet supported \ + (sled {sled_id} has {count}; \ + planner wants {desired_zone_count})" + ))); + } + }; + + for i in count..desired_zone_count { + self.sled_add_zone_internal_dns( + sled_id, + i.try_into().map_err(|_| { + Error::Planner(anyhow!("zone index overflow")) + })?, + )?; + } + + Ok(EnsureMultiple::Changed { added: to_add, removed: 0 }) + } + pub fn sled_ensure_zone_ntp( &mut self, sled_id: SledUuid, @@ -636,14 +714,18 @@ impl<'a> BlueprintBuilder<'a> { let sled_subnet = sled_info.subnet; let ip = self.sled_alloc_ip(sled_id)?; let ntp_address = SocketAddrV6::new(ip, NTP_PORT, 0, 0); + // Construct the list of internal DNS servers. // // It'd be tempting to get this list from the other internal NTP - // servers but there may not be any of those. We could also - // construct this list manually from the set of internal DNS servers - // actually deployed. Instead, we take the same approach as RSS: - // these are at known, fixed addresses relative to the AZ subnet - // (which itself is a known-prefix parent subnet of the sled subnet). + // servers, but there may not be any of those. We could also + // construct it manually from the set of internal DNS servers + // actually deployed, or ask the DNS subnet allocator; but those + // would both require that all the internal DNS zones be added + // before any NTP zones, a constraint we don't currently enforce. + // Instead, we take the same approach as RSS: they are at known, + // fixed addresses relative to the AZ subnet (which itself is a + // known-prefix parent subnet of the sled subnet). let dns_servers = get_internal_dns_server_addresses(sled_subnet.net().prefix()); @@ -1139,13 +1221,13 @@ impl<'a> BlueprintBuilder<'a> { allocator.alloc().ok_or(Error::OutOfAddresses { sled_id }) } - // Selects a zpools for this zone type. - // - // This zpool may be used for either durable storage or transient - // storage - the usage is up to the caller. - // - // If `zone_kind` already exists on `sled_id`, it is prevented - // from using the same zpool as exisitng zones with the same kind. + /// Selects a zpool for this zone type. + /// + /// This zpool may be used for either durable storage or transient + /// storage - the usage is up to the caller. + /// + /// If `zone_kind` already exists on `sled_id`, it is prevented + /// from using the same zpool as existing zones with the same kind. fn sled_select_zpool( &self, sled_id: SledUuid, diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/internal_dns.rs b/nexus/reconfigurator/planning/src/blueprint_builder/internal_dns.rs new file mode 100644 index 0000000000..61b4ec64de --- /dev/null +++ b/nexus/reconfigurator/planning/src/blueprint_builder/internal_dns.rs @@ -0,0 +1,183 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::Error; +use nexus_types::deployment::blueprint_zone_type::InternalDns; +use nexus_types::deployment::Blueprint; +use nexus_types::deployment::BlueprintZoneFilter; +use nexus_types::deployment::BlueprintZoneType; +use nexus_types::deployment::PlanningInput; +use omicron_common::address::DnsSubnet; +use omicron_common::address::ReservedRackSubnet; +use omicron_common::policy::MAX_INTERNAL_DNS_REDUNDANCY; +use std::collections::BTreeSet; + +/// Internal DNS zones are not allocated an address in the sled's subnet. +/// Instead, they get a /64 subnet of the "reserved" rack subnet (so that +/// it's routable with IPv6), and use the first address in that. There may +/// be at most `MAX_INTERNAL_DNS_REDUNDANCY` subnets (and so servers) +/// allocated. This structure tracks which subnets are currently allocated. +#[derive(Debug)] +pub struct DnsSubnetAllocator { + in_use: BTreeSet, +} + +impl DnsSubnetAllocator { + pub fn new<'a>( + parent_blueprint: &'a Blueprint, + input: &'a PlanningInput, + ) -> Result { + let in_use = parent_blueprint + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + .filter_map(|(_sled_id, zone_config)| match zone_config.zone_type { + BlueprintZoneType::InternalDns(InternalDns { + dns_address, + .. + }) => Some(DnsSubnet::from_addr(*dns_address.ip())), + _ => None, + }) + .collect::>(); + + let redundancy = input.target_internal_dns_zone_count(); + if redundancy > MAX_INTERNAL_DNS_REDUNDANCY { + return Err(Error::TooManyDnsServers); + } + + Ok(Self { in_use }) + } + + /// Allocate the first available DNS subnet, or call a function to generate + /// a default. The default is needed because we can't necessarily guess the + /// correct reserved rack subnet (e.g., there might not be any internal DNS + /// zones in the parent blueprint, though that would itself be odd), but we + /// can derive it at runtime from the sled address. + pub fn alloc( + &mut self, + rack_subnet: ReservedRackSubnet, + ) -> Result { + let new = if let Some(first) = self.in_use.first() { + // Take the first available DNS subnet. We currently generate + // all `MAX_INTERNAL_DNS_REDUNDANCY` subnets and subtract any + // that are in use; this is fine as long as that constant is small. + let subnets = BTreeSet::from_iter( + ReservedRackSubnet::from_subnet(first.subnet()) + .get_dns_subnets(), + ); + let mut avail = subnets.difference(&self.in_use); + if let Some(first) = avail.next() { + *first + } else { + return Err(Error::NoAvailableDnsSubnets); + } + } else { + rack_subnet.get_dns_subnet(1) + }; + self.in_use.insert(new); + Ok(new) + } + + #[cfg(test)] + fn first(&self) -> Option { + self.in_use.first().copied() + } + + #[cfg(test)] + fn pop_first(&mut self) -> Option { + self.in_use.pop_first() + } + + #[cfg(test)] + fn last(&self) -> Option { + self.in_use.last().cloned() + } + + #[cfg(test)] + fn len(&self) -> usize { + self.in_use.len() + } +} + +#[cfg(test)] +pub mod test { + use super::*; + use crate::blueprint_builder::test::verify_blueprint; + use crate::example::ExampleSystem; + use omicron_common::policy::{ + INTERNAL_DNS_REDUNDANCY, MAX_INTERNAL_DNS_REDUNDANCY, + }; + use omicron_test_utils::dev::test_setup_log; + + #[test] + fn test_dns_subnet_allocator() { + static TEST_NAME: &str = "test_dns_subnet_allocator"; + let logctx = test_setup_log(TEST_NAME); + + // Use our example system to create a blueprint and input. + let example = + ExampleSystem::new(&logctx.log, TEST_NAME, INTERNAL_DNS_REDUNDANCY); + let blueprint1 = &example.blueprint; + verify_blueprint(blueprint1); + + // Create an allocator. + let mut allocator = DnsSubnetAllocator::new(blueprint1, &example.input) + .expect("can't create allocator"); + + // Save the first & last allocated subnets. + let first = allocator.first().expect("should be a first subnet"); + let last = allocator.last().expect("should be a last subnet"); + assert!(last > first, "first should come before last"); + + // Derive the reserved rack subnet. + let rack_subnet = first.rack_subnet(); + assert_eq!( + rack_subnet, + last.rack_subnet(), + "first & last DNS subnets should be in the same rack subnet" + ); + + // Allocate two new subnets. + assert_eq!(MAX_INTERNAL_DNS_REDUNDANCY - INTERNAL_DNS_REDUNDANCY, 2); + assert_eq!( + allocator.len(), + INTERNAL_DNS_REDUNDANCY, + "should be {INTERNAL_DNS_REDUNDANCY} subnets allocated" + ); + let new1 = + allocator.alloc(rack_subnet).expect("failed to allocate a subnet"); + let new2 = allocator + .alloc(rack_subnet) + .expect("failed to allocate another subnet"); + assert!( + new1 > last, + "newly allocated subnets should be after initial ones" + ); + assert!(new2 > new1, "allocated subnets out of order"); + assert_ne!(new1, new2, "allocated duplicate subnets"); + assert_eq!( + allocator.len(), + MAX_INTERNAL_DNS_REDUNDANCY, + "should be {INTERNAL_DNS_REDUNDANCY} subnets allocated" + ); + allocator.alloc(rack_subnet).expect_err("no subnets available"); + + // Test packing. + let first = allocator.pop_first().expect("should be a first subnet"); + let second = allocator.pop_first().expect("should be a second subnet"); + assert!(first < second, "first should be before second"); + assert_eq!( + allocator.alloc(rack_subnet).expect("allocation failed"), + first, + "should get first subnet" + ); + assert_eq!( + allocator.alloc(rack_subnet).expect("allocation failed"), + second, + "should get second subnet" + ); + allocator.alloc(rack_subnet).expect_err("no subnets available"); + + // Done! + logctx.cleanup_successful(); + } +} diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/mod.rs b/nexus/reconfigurator/planning/src/blueprint_builder/mod.rs index 99d3b41772..9c6e51f1de 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/mod.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/mod.rs @@ -6,6 +6,7 @@ mod builder; mod external_networking; +mod internal_dns; mod zones; pub use builder::*; diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index e52fe3fc4b..86aa00fb52 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -79,6 +79,9 @@ impl ExampleSystem { vec![], ) .unwrap(); + let _ = builder + .sled_ensure_zone_multiple_internal_dns(sled_id, 1) + .unwrap(); let _ = builder.sled_ensure_disks(sled_id, sled_resources).unwrap(); for pool_name in sled_resources.zpools.keys() { let _ = builder diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 3bd1b8757e..7149aecb85 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -212,10 +212,7 @@ impl<'a> Planner<'a> { fn do_plan_add(&mut self) -> Result<(), Error> { // Internal DNS is a prerequisite for bringing up all other zones. At // this point, we assume that internal DNS (as a service) is already - // functioning. At some point, this function will have to grow the - // ability to determine whether more internal DNS zones need to be - // added and where they should go. And the blueprint builder will need - // to grow the ability to provision one. + // functioning. // After we make our initial pass through the sleds below to check for // zones every sled should have (NTP, Crucible), we'll start making @@ -356,6 +353,7 @@ impl<'a> Planner<'a> { for zone_kind in [ DiscretionaryOmicronZone::BoundaryNtp, DiscretionaryOmicronZone::CockroachDb, + DiscretionaryOmicronZone::InternalDns, DiscretionaryOmicronZone::Nexus, ] { let num_zones_to_add = self.num_additional_zones_needed(zone_kind); @@ -434,6 +432,9 @@ impl<'a> Planner<'a> { DiscretionaryOmicronZone::CockroachDb => { self.input.target_cockroachdb_zone_count() } + DiscretionaryOmicronZone::InternalDns => { + self.input.target_internal_dns_zone_count() + } DiscretionaryOmicronZone::Nexus => { self.input.target_nexus_zone_count() } @@ -516,6 +517,12 @@ impl<'a> Planner<'a> { new_total_zone_count, )? } + DiscretionaryOmicronZone::InternalDns => { + self.blueprint.sled_ensure_zone_multiple_internal_dns( + sled_id, + new_total_zone_count, + )? + } DiscretionaryOmicronZone::Nexus => { self.blueprint.sled_ensure_zone_multiple_nexus( sled_id, @@ -745,6 +752,7 @@ mod test { use nexus_types::inventory::OmicronZonesFound; use omicron_common::api::external::Generation; use omicron_common::disk::DiskIdentity; + use omicron_common::policy::MAX_INTERNAL_DNS_REDUNDANCY; use omicron_test_utils::dev::test_setup_log; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::PhysicalDiskUuid; @@ -1029,6 +1037,11 @@ mod test { // one sled we have. let mut builder = input.into_builder(); builder.policy_mut().target_nexus_zone_count = 5; + + // But we don't want it to add any more internal DNS zones, + // which it would by default (because we have only one sled). + builder.policy_mut().target_internal_dns_zone_count = 1; + let input = builder.build(); let blueprint2 = Planner::new_based_on( logctx.log.clone(), @@ -1141,6 +1154,101 @@ mod test { logctx.cleanup_successful(); } + /// Check that the planner will spread additional internal DNS zones out across + /// sleds as it adds them + #[test] + fn test_spread_internal_dns_zones_across_sleds() { + static TEST_NAME: &str = + "planner_spread_internal_dns_zones_across_sleds"; + let logctx = test_setup_log(TEST_NAME); + + // Use our example system as a starting point. + let (collection, input, blueprint1) = + example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + + // This blueprint should have exactly 3 internal DNS zones: one on each sled. + assert_eq!(blueprint1.blueprint_zones.len(), 3); + for sled_config in blueprint1.blueprint_zones.values() { + assert_eq!( + sled_config + .zones + .iter() + .filter(|z| z.zone_type.is_internal_dns()) + .count(), + 1 + ); + } + + // Try to run the planner with a high number of internal DNS zones; + // it will fail because the target is > MAX_DNS_REDUNDANCY. + let mut builder = input.clone().into_builder(); + builder.policy_mut().target_internal_dns_zone_count = 14; + assert!( + Planner::new_based_on( + logctx.log.clone(), + &blueprint1, + &builder.build(), + "test_blueprint2", + &collection, + ) + .is_err(), + "too many DNS zones" + ); + + // Try again with a reasonable number. + let mut builder = input.into_builder(); + builder.policy_mut().target_internal_dns_zone_count = + MAX_INTERNAL_DNS_REDUNDANCY; + let blueprint2 = Planner::new_based_on( + logctx.log.clone(), + &blueprint1, + &builder.build(), + "test_blueprint2", + &collection, + ) + .expect("failed to create planner") + .with_rng_seed((TEST_NAME, "bp2")) + .plan() + .expect("failed to plan"); + + let diff = blueprint2.diff_since_blueprint(&blueprint1); + println!( + "1 -> 2 (added additional internal DNS zones):\n{}", + diff.display() + ); + assert_eq!(diff.sleds_added.len(), 0); + assert_eq!(diff.sleds_removed.len(), 0); + assert_eq!(diff.sleds_modified.len(), 2); + + // 2 sleds should each get 1 additional internal DNS zone. + let mut total_new_zones = 0; + for sled_id in diff.sleds_modified { + assert!(!diff.zones.removed.contains_key(&sled_id)); + assert!(!diff.zones.modified.contains_key(&sled_id)); + if let Some(zones_added) = &diff.zones.added.get(&sled_id) { + let zones = &zones_added.zones; + match zones.len() { + n @ 1 => { + total_new_zones += n; + } + n => { + panic!("unexpected number of zones added to {sled_id}: {n}") + } + } + for zone in zones { + assert_eq!( + zone.kind(), + ZoneKind::InternalDns, + "unexpectedly added a non-internal-DNS zone: {zone:?}" + ); + } + } + } + assert_eq!(total_new_zones, 2); + + logctx.cleanup_successful(); + } + #[test] fn test_crucible_allocation_skips_nonprovisionable_disks() { static TEST_NAME: &str = @@ -1153,9 +1261,10 @@ mod test { let mut builder = input.into_builder(); - // Avoid churning on the quantity of Nexus zones - we're okay staying at - // one. + // Avoid churning on the quantity of Nexus and internal DNS zones - + // we're okay staying at one each. builder.policy_mut().target_nexus_zone_count = 1; + builder.policy_mut().target_internal_dns_zone_count = 1; // Make generated disk ids deterministic let mut disk_rng = @@ -1236,9 +1345,10 @@ mod test { let mut builder = input.into_builder(); - // Aside: Avoid churning on the quantity of Nexus zones - we're okay - // staying at one. + // Avoid churning on the quantity of Nexus and internal DNS zones - + // we're okay staying at one each. builder.policy_mut().target_nexus_zone_count = 1; + builder.policy_mut().target_internal_dns_zone_count = 1; // The example system should be assigning crucible zones to each // in-service disk. When we expunge one of these disks, the planner diff --git a/nexus/reconfigurator/planning/src/planner/omicron_zone_placement.rs b/nexus/reconfigurator/planning/src/planner/omicron_zone_placement.rs index 2fb60e66f8..6f3bac0ecc 100644 --- a/nexus/reconfigurator/planning/src/planner/omicron_zone_placement.rs +++ b/nexus/reconfigurator/planning/src/planner/omicron_zone_placement.rs @@ -16,6 +16,7 @@ use std::mem; pub(crate) enum DiscretionaryOmicronZone { BoundaryNtp, CockroachDb, + InternalDns, Nexus, // TODO expand this enum as we start to place more services } @@ -27,6 +28,7 @@ impl DiscretionaryOmicronZone { match zone_type { BlueprintZoneType::BoundaryNtp(_) => Some(Self::BoundaryNtp), BlueprintZoneType::CockroachDb(_) => Some(Self::CockroachDb), + BlueprintZoneType::InternalDns(_) => Some(Self::InternalDns), BlueprintZoneType::Nexus(_) => Some(Self::Nexus), // Zones that we should place but don't yet. BlueprintZoneType::Clickhouse(_) @@ -34,7 +36,6 @@ impl DiscretionaryOmicronZone { | BlueprintZoneType::ClickhouseServer(_) | BlueprintZoneType::CruciblePantry(_) | BlueprintZoneType::ExternalDns(_) - | BlueprintZoneType::InternalDns(_) | BlueprintZoneType::Oximeter(_) => None, // Zones that get special handling for placement (all sleds get // them, although internal NTP has some interactions with boundary @@ -50,6 +51,7 @@ impl From for ZoneKind { match zone { DiscretionaryOmicronZone::BoundaryNtp => Self::BoundaryNtp, DiscretionaryOmicronZone::CockroachDb => Self::CockroachDb, + DiscretionaryOmicronZone::InternalDns => Self::InternalDns, DiscretionaryOmicronZone::Nexus => Self::Nexus, } } diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 7298db7a73..26e4910c5b 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -39,6 +39,7 @@ use omicron_common::api::external::ByteCount; use omicron_common::api::external::Generation; use omicron_common::disk::DiskIdentity; use omicron_common::disk::DiskVariant; +use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_common::policy::NEXUS_REDUNDANCY; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::PhysicalDiskUuid; @@ -81,6 +82,7 @@ pub struct SystemDescription { available_scrimlet_slots: BTreeSet, target_boundary_ntp_zone_count: usize, target_nexus_zone_count: usize, + target_internal_dns_zone_count: usize, target_cockroachdb_zone_count: usize, target_cockroachdb_cluster_version: CockroachDbClusterVersion, service_ip_pool_ranges: Vec, @@ -130,6 +132,7 @@ impl SystemDescription { // Policy defaults let target_nexus_zone_count = NEXUS_REDUNDANCY; + let target_internal_dns_zone_count = INTERNAL_DNS_REDUNDANCY; // TODO-cleanup These are wrong, but we don't currently set up any // boundary NTP or CRDB nodes in our fake system, so this prevents @@ -156,6 +159,7 @@ impl SystemDescription { available_scrimlet_slots, target_boundary_ntp_zone_count, target_nexus_zone_count, + target_internal_dns_zone_count, target_cockroachdb_zone_count, target_cockroachdb_cluster_version, service_ip_pool_ranges, @@ -325,6 +329,7 @@ impl SystemDescription { service_ip_pool_ranges: self.service_ip_pool_ranges.clone(), target_boundary_ntp_zone_count: self.target_boundary_ntp_zone_count, target_nexus_zone_count: self.target_nexus_zone_count, + target_internal_dns_zone_count: self.target_internal_dns_zone_count, target_cockroachdb_zone_count: self.target_cockroachdb_zone_count, target_cockroachdb_cluster_version: self .target_cockroachdb_cluster_version, diff --git a/nexus/reconfigurator/planning/tests/output/blueprint_builder_initial_diff.txt b/nexus/reconfigurator/planning/tests/output/blueprint_builder_initial_diff.txt index 01b7ceb46b..08acad5a45 100644 --- a/nexus/reconfigurator/planning/tests/output/blueprint_builder_initial_diff.txt +++ b/nexus/reconfigurator/planning/tests/output/blueprint_builder_initial_diff.txt @@ -24,16 +24,17 @@ to: blueprint e4aeb3b3-272f-4967-be34-2d34daa46aa1 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 44afce85-3377-4b20-a398-517c1579df4d in service fd00:1122:3344:103::23 - crucible 4644ea0c-0ec3-41be-a356-660308e1c3fc in service fd00:1122:3344:103::2c - crucible 55f4d117-0b9d-4256-a2c0-f46d3ed5fff9 in service fd00:1122:3344:103::25 - crucible 5c6a4628-8831-483b-995f-79b9126c4d04 in service fd00:1122:3344:103::28 - crucible 6a01210c-45ed-41a5-9230-8e05ecf5dd8f in service fd00:1122:3344:103::29 - crucible 7004cab9-dfc0-43ba-92d3-58d4ced66025 in service fd00:1122:3344:103::24 - crucible 79552859-fbd3-43bb-a9d3-6baba25558f8 in service fd00:1122:3344:103::26 - crucible 90696819-9b53-485a-9c65-ca63602e843e in service fd00:1122:3344:103::27 - crucible c99525b3-3680-4df6-9214-2ee3e1020e8b in service fd00:1122:3344:103::2a - crucible f42959d3-9eef-4e3b-b404-6177ce3ec7a1 in service fd00:1122:3344:103::2b + crucible 38b047ea-e3de-4859-b8e0-70cac5871446 in service fd00:1122:3344:103::2c + crucible 4644ea0c-0ec3-41be-a356-660308e1c3fc in service fd00:1122:3344:103::2b + crucible 55f4d117-0b9d-4256-a2c0-f46d3ed5fff9 in service fd00:1122:3344:103::24 + crucible 5c6a4628-8831-483b-995f-79b9126c4d04 in service fd00:1122:3344:103::27 + crucible 6a01210c-45ed-41a5-9230-8e05ecf5dd8f in service fd00:1122:3344:103::28 + crucible 7004cab9-dfc0-43ba-92d3-58d4ced66025 in service fd00:1122:3344:103::23 + crucible 79552859-fbd3-43bb-a9d3-6baba25558f8 in service fd00:1122:3344:103::25 + crucible 90696819-9b53-485a-9c65-ca63602e843e in service fd00:1122:3344:103::26 + crucible c99525b3-3680-4df6-9214-2ee3e1020e8b in service fd00:1122:3344:103::29 + crucible f42959d3-9eef-4e3b-b404-6177ce3ec7a1 in service fd00:1122:3344:103::2a + internal_dns 44afce85-3377-4b20-a398-517c1579df4d in service fd00:1122:3344:1::1 internal_ntp c81c9d4a-36d7-4796-9151-f564d3735152 in service fd00:1122:3344:103::21 nexus b2573120-9c91-4ed7-8b4f-a7bfe8dbc807 in service fd00:1122:3344:103::22 @@ -60,18 +61,19 @@ to: blueprint e4aeb3b3-272f-4967-be34-2d34daa46aa1 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 0faa9350-2c02-47c7-a0a6-9f4afd69152c in service fd00:1122:3344:101::2c - crucible 5b44003e-1a3d-4152-b606-872c72efce0e in service fd00:1122:3344:101::25 - crucible 943fea7a-9458-4935-9dc7-01ee5cfe5a02 in service fd00:1122:3344:101::29 - crucible 95c3b6d1-2592-4252-b5c1-5d0faf3ce9c9 in service fd00:1122:3344:101::24 - crucible a5a0b7a9-37c9-4dbd-8393-ec7748ada3b0 in service fd00:1122:3344:101::2b - crucible a9a6a974-8953-4783-b815-da46884f2c02 in service fd00:1122:3344:101::23 - crucible aa25add8-60b0-4ace-ac60-15adcdd32d50 in service fd00:1122:3344:101::2a - crucible b6f2dd1e-7f98-4a68-9df2-b33c69d1f7ea in service fd00:1122:3344:101::27 - crucible dc22d470-dc46-436b-9750-25c8d7d369e2 in service fd00:1122:3344:101::26 - crucible f7e434f9-6d4a-476b-a9e2-48d6ee28a08e in service fd00:1122:3344:101::28 - internal_ntp 38b047ea-e3de-4859-b8e0-70cac5871446 in service fd00:1122:3344:101::21 - nexus fb36b9dc-273a-4bc3-aaa9-19ee4d0ef552 in service fd00:1122:3344:101::22 + crucible 0faa9350-2c02-47c7-a0a6-9f4afd69152c in service fd00:1122:3344:101::2a + crucible 29278a22-1ba1-4117-bfdb-39fcb9ae7fd1 in service fd00:1122:3344:101::2c + crucible 5b44003e-1a3d-4152-b606-872c72efce0e in service fd00:1122:3344:101::23 + crucible 943fea7a-9458-4935-9dc7-01ee5cfe5a02 in service fd00:1122:3344:101::27 + crucible a5a0b7a9-37c9-4dbd-8393-ec7748ada3b0 in service fd00:1122:3344:101::29 + crucible aa25add8-60b0-4ace-ac60-15adcdd32d50 in service fd00:1122:3344:101::28 + crucible aac3ab51-9e2b-4605-9bf6-e3eb3681c2b5 in service fd00:1122:3344:101::2b + crucible b6f2dd1e-7f98-4a68-9df2-b33c69d1f7ea in service fd00:1122:3344:101::25 + crucible dc22d470-dc46-436b-9750-25c8d7d369e2 in service fd00:1122:3344:101::24 + crucible f7e434f9-6d4a-476b-a9e2-48d6ee28a08e in service fd00:1122:3344:101::26 + internal_dns 95c3b6d1-2592-4252-b5c1-5d0faf3ce9c9 in service fd00:1122:3344:2::1 + internal_ntp fb36b9dc-273a-4bc3-aaa9-19ee4d0ef552 in service fd00:1122:3344:101::21 + nexus a9a6a974-8953-4783-b815-da46884f2c02 in service fd00:1122:3344:101::22 sled be7f4375-2a6b-457f-b1a4-3074a715e5fe: @@ -96,18 +98,19 @@ to: blueprint e4aeb3b3-272f-4967-be34-2d34daa46aa1 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 248db330-56e6-4c7e-b5ff-9cd6cbcb210a in service fd00:1122:3344:102::2c - crucible 353b0aff-4c71-4fae-a6bd-adcb1d2a1a1d in service fd00:1122:3344:102::29 - crucible 4330134c-41b9-4097-aa0b-3eaefa06d473 in service fd00:1122:3344:102::24 - crucible 65d03287-e43f-45f4-902e-0a5e4638f31a in service fd00:1122:3344:102::25 - crucible 6a5901b1-f9d7-425c-8ecb-a786c900f217 in service fd00:1122:3344:102::27 - crucible 9b722fea-a186-4bc3-bc37-ce7f6de6a796 in service fd00:1122:3344:102::23 - crucible b3583b5f-4a62-4471-9be7-41e61578de4c in service fd00:1122:3344:102::2a - crucible bac92034-b9e6-4e8b-9ffb-dbba9caec88d in service fd00:1122:3344:102::28 - crucible d9653001-f671-4905-a410-6a7abc358318 in service fd00:1122:3344:102::2b - crucible edaca77e-5806-446a-b00c-125962cd551d in service fd00:1122:3344:102::26 - internal_ntp aac3ab51-9e2b-4605-9bf6-e3eb3681c2b5 in service fd00:1122:3344:102::21 - nexus 29278a22-1ba1-4117-bfdb-39fcb9ae7fd1 in service fd00:1122:3344:102::22 + crucible 248db330-56e6-4c7e-b5ff-9cd6cbcb210a in service fd00:1122:3344:102::29 + crucible 353b0aff-4c71-4fae-a6bd-adcb1d2a1a1d in service fd00:1122:3344:102::26 + crucible 6a5901b1-f9d7-425c-8ecb-a786c900f217 in service fd00:1122:3344:102::24 + crucible b3583b5f-4a62-4471-9be7-41e61578de4c in service fd00:1122:3344:102::27 + crucible b97bdef5-ed14-4e11-9d3b-3379c18ea694 in service fd00:1122:3344:102::2c + crucible bac92034-b9e6-4e8b-9ffb-dbba9caec88d in service fd00:1122:3344:102::25 + crucible c240ec8c-cec5-4117-944d-faeb5672d568 in service fd00:1122:3344:102::2b + crucible cf766535-9b6f-4263-a83a-86f45f7b005b in service fd00:1122:3344:102::2a + crucible d9653001-f671-4905-a410-6a7abc358318 in service fd00:1122:3344:102::28 + crucible edaca77e-5806-446a-b00c-125962cd551d in service fd00:1122:3344:102::23 + internal_dns 65d03287-e43f-45f4-902e-0a5e4638f31a in service fd00:1122:3344:3::1 + internal_ntp 9b722fea-a186-4bc3-bc37-ce7f6de6a796 in service fd00:1122:3344:102::21 + nexus 4330134c-41b9-4097-aa0b-3eaefa06d473 in service fd00:1122:3344:102::22 COCKROACHDB SETTINGS: diff --git a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt index 3b14db49c7..4ceb76ba39 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt @@ -25,16 +25,17 @@ to: blueprint 4171ad05-89dd-474b-846b-b007e4346366 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 322ee9f1-8903-4542-a0a8-a54cefabdeca in service fd00:1122:3344:103::24 - crucible 4ab1650f-32c5-447f-939d-64b8103a7645 in service fd00:1122:3344:103::2a - crucible 64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service fd00:1122:3344:103::27 - crucible 6e811d86-8aa7-4660-935b-84b4b7721b10 in service fd00:1122:3344:103::2b - crucible 747d2426-68bf-4c22-8806-41d290b5d5f5 in service fd00:1122:3344:103::25 - crucible 7fbd2c38-5dc3-48c4-b061-558a2041d70f in service fd00:1122:3344:103::2c - crucible 8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service fd00:1122:3344:103::29 - crucible b14d5478-1a0e-4b90-b526-36b06339dfc4 in service fd00:1122:3344:103::28 - crucible b40f7c7b-526c-46c8-ae33-67280c280eb7 in service fd00:1122:3344:103::23 - crucible be97b92b-38d6-422a-8c76-d37060f75bd2 in service fd00:1122:3344:103::26 + crucible 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service fd00:1122:3344:103::2c + crucible 322ee9f1-8903-4542-a0a8-a54cefabdeca in service fd00:1122:3344:103::23 + crucible 4ab1650f-32c5-447f-939d-64b8103a7645 in service fd00:1122:3344:103::29 + crucible 64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service fd00:1122:3344:103::26 + crucible 6e811d86-8aa7-4660-935b-84b4b7721b10 in service fd00:1122:3344:103::2a + crucible 747d2426-68bf-4c22-8806-41d290b5d5f5 in service fd00:1122:3344:103::24 + crucible 7fbd2c38-5dc3-48c4-b061-558a2041d70f in service fd00:1122:3344:103::2b + crucible 8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service fd00:1122:3344:103::28 + crucible b14d5478-1a0e-4b90-b526-36b06339dfc4 in service fd00:1122:3344:103::27 + crucible be97b92b-38d6-422a-8c76-d37060f75bd2 in service fd00:1122:3344:103::25 + internal_dns b40f7c7b-526c-46c8-ae33-67280c280eb7 in service fd00:1122:3344:1::1 internal_ntp 267ed614-92af-4b9d-bdba-c2881c2e43a2 in service fd00:1122:3344:103::21 nexus cc816cfe-3869-4dde-b596-397d41198628 in service fd00:1122:3344:103::22 @@ -61,18 +62,19 @@ to: blueprint 4171ad05-89dd-474b-846b-b007e4346366 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 02acbe6a-1c88-47e3-94c3-94084cbde098 in service fd00:1122:3344:101::27 - crucible 07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service fd00:1122:3344:101::26 - crucible 10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service fd00:1122:3344:101::24 - crucible 2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service fd00:1122:3344:101::29 - crucible 3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service fd00:1122:3344:101::23 - crucible 587be699-a320-4c79-b320-128d9ecddc0b in service fd00:1122:3344:101::2b - crucible 6fa06115-4959-4913-8e7b-dd70d7651f07 in service fd00:1122:3344:101::2c - crucible 8f3a1cc5-9195-4a30-ad02-b804278fe639 in service fd00:1122:3344:101::28 - crucible a1696cd4-588c-484a-b95b-66e824c0ce05 in service fd00:1122:3344:101::25 - crucible a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service fd00:1122:3344:101::2a - internal_ntp 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service fd00:1122:3344:101::21 - nexus c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service fd00:1122:3344:101::22 + crucible 02acbe6a-1c88-47e3-94c3-94084cbde098 in service fd00:1122:3344:101::25 + crucible 07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service fd00:1122:3344:101::24 + crucible 2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service fd00:1122:3344:101::27 + crucible 47199d48-534c-4267-a654-d2d90e64b498 in service fd00:1122:3344:101::2b + crucible 587be699-a320-4c79-b320-128d9ecddc0b in service fd00:1122:3344:101::29 + crucible 6fa06115-4959-4913-8e7b-dd70d7651f07 in service fd00:1122:3344:101::2a + crucible 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service fd00:1122:3344:101::2c + crucible 8f3a1cc5-9195-4a30-ad02-b804278fe639 in service fd00:1122:3344:101::26 + crucible a1696cd4-588c-484a-b95b-66e824c0ce05 in service fd00:1122:3344:101::23 + crucible a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service fd00:1122:3344:101::28 + internal_dns 10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service fd00:1122:3344:2::1 + internal_ntp c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service fd00:1122:3344:101::21 + nexus 3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service fd00:1122:3344:101::22 sled 590e3034-d946-4166-b0e5-2d0034197a07: @@ -97,18 +99,19 @@ to: blueprint 4171ad05-89dd-474b-846b-b007e4346366 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 18f8fe40-646e-4962-b17a-20e201f3a6e5 in service fd00:1122:3344:102::2a - crucible 56d5d7cf-db2c-40a3-a775-003241ad4820 in service fd00:1122:3344:102::29 - crucible 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service fd00:1122:3344:102::2b - crucible 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service fd00:1122:3344:102::26 - crucible 93f2f40c-5616-4d8d-8519-ec6debdcede0 in service fd00:1122:3344:102::2c - crucible ab7ba6df-d401-40bd-940e-faf57c57aa2a in service fd00:1122:3344:102::28 - crucible af322036-371f-437c-8c08-7f40f3f1403b in service fd00:1122:3344:102::23 - crucible d637264f-6f40-44c2-8b7e-a179430210d2 in service fd00:1122:3344:102::25 - crucible dce226c9-7373-4bfa-8a94-79dc472857a6 in service fd00:1122:3344:102::27 - crucible edabedf3-839c-488d-ad6f-508ffa864674 in service fd00:1122:3344:102::24 - internal_ntp 47199d48-534c-4267-a654-d2d90e64b498 in service fd00:1122:3344:102::21 - nexus 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service fd00:1122:3344:102::22 + crucible 0565e7e4-f13a-4123-8928-d715f83e36aa in service fd00:1122:3344:102::2b + crucible 18f8fe40-646e-4962-b17a-20e201f3a6e5 in service fd00:1122:3344:102::27 + crucible 1cc3f503-2001-4d85-80e5-c7c40d2e3b10 in service fd00:1122:3344:102::2a + crucible 56d5d7cf-db2c-40a3-a775-003241ad4820 in service fd00:1122:3344:102::26 + crucible 62058f4c-c747-4e21-a8dc-2fd4a160c98c in service fd00:1122:3344:102::2c + crucible 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service fd00:1122:3344:102::28 + crucible 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service fd00:1122:3344:102::23 + crucible 93f2f40c-5616-4d8d-8519-ec6debdcede0 in service fd00:1122:3344:102::29 + crucible ab7ba6df-d401-40bd-940e-faf57c57aa2a in service fd00:1122:3344:102::25 + crucible dce226c9-7373-4bfa-8a94-79dc472857a6 in service fd00:1122:3344:102::24 + internal_dns d637264f-6f40-44c2-8b7e-a179430210d2 in service fd00:1122:3344:3::1 + internal_ntp af322036-371f-437c-8c08-7f40f3f1403b in service fd00:1122:3344:102::21 + nexus edabedf3-839c-488d-ad6f-508ffa864674 in service fd00:1122:3344:102::22 ADDED SLEDS: diff --git a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt index b252a21d7d..dde82c189f 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt @@ -25,16 +25,17 @@ to: blueprint f432fcd5-1284-4058-8b4a-9286a3de6163 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 322ee9f1-8903-4542-a0a8-a54cefabdeca in service fd00:1122:3344:103::24 - crucible 4ab1650f-32c5-447f-939d-64b8103a7645 in service fd00:1122:3344:103::2a - crucible 64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service fd00:1122:3344:103::27 - crucible 6e811d86-8aa7-4660-935b-84b4b7721b10 in service fd00:1122:3344:103::2b - crucible 747d2426-68bf-4c22-8806-41d290b5d5f5 in service fd00:1122:3344:103::25 - crucible 7fbd2c38-5dc3-48c4-b061-558a2041d70f in service fd00:1122:3344:103::2c - crucible 8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service fd00:1122:3344:103::29 - crucible b14d5478-1a0e-4b90-b526-36b06339dfc4 in service fd00:1122:3344:103::28 - crucible b40f7c7b-526c-46c8-ae33-67280c280eb7 in service fd00:1122:3344:103::23 - crucible be97b92b-38d6-422a-8c76-d37060f75bd2 in service fd00:1122:3344:103::26 + crucible 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service fd00:1122:3344:103::2c + crucible 322ee9f1-8903-4542-a0a8-a54cefabdeca in service fd00:1122:3344:103::23 + crucible 4ab1650f-32c5-447f-939d-64b8103a7645 in service fd00:1122:3344:103::29 + crucible 64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service fd00:1122:3344:103::26 + crucible 6e811d86-8aa7-4660-935b-84b4b7721b10 in service fd00:1122:3344:103::2a + crucible 747d2426-68bf-4c22-8806-41d290b5d5f5 in service fd00:1122:3344:103::24 + crucible 7fbd2c38-5dc3-48c4-b061-558a2041d70f in service fd00:1122:3344:103::2b + crucible 8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service fd00:1122:3344:103::28 + crucible b14d5478-1a0e-4b90-b526-36b06339dfc4 in service fd00:1122:3344:103::27 + crucible be97b92b-38d6-422a-8c76-d37060f75bd2 in service fd00:1122:3344:103::25 + internal_dns b40f7c7b-526c-46c8-ae33-67280c280eb7 in service fd00:1122:3344:1::1 internal_ntp 267ed614-92af-4b9d-bdba-c2881c2e43a2 in service fd00:1122:3344:103::21 nexus cc816cfe-3869-4dde-b596-397d41198628 in service fd00:1122:3344:103::22 @@ -61,18 +62,19 @@ to: blueprint f432fcd5-1284-4058-8b4a-9286a3de6163 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 02acbe6a-1c88-47e3-94c3-94084cbde098 in service fd00:1122:3344:101::27 - crucible 07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service fd00:1122:3344:101::26 - crucible 10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service fd00:1122:3344:101::24 - crucible 2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service fd00:1122:3344:101::29 - crucible 3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service fd00:1122:3344:101::23 - crucible 587be699-a320-4c79-b320-128d9ecddc0b in service fd00:1122:3344:101::2b - crucible 6fa06115-4959-4913-8e7b-dd70d7651f07 in service fd00:1122:3344:101::2c - crucible 8f3a1cc5-9195-4a30-ad02-b804278fe639 in service fd00:1122:3344:101::28 - crucible a1696cd4-588c-484a-b95b-66e824c0ce05 in service fd00:1122:3344:101::25 - crucible a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service fd00:1122:3344:101::2a - internal_ntp 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service fd00:1122:3344:101::21 - nexus c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service fd00:1122:3344:101::22 + crucible 02acbe6a-1c88-47e3-94c3-94084cbde098 in service fd00:1122:3344:101::25 + crucible 07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service fd00:1122:3344:101::24 + crucible 2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service fd00:1122:3344:101::27 + crucible 47199d48-534c-4267-a654-d2d90e64b498 in service fd00:1122:3344:101::2b + crucible 587be699-a320-4c79-b320-128d9ecddc0b in service fd00:1122:3344:101::29 + crucible 6fa06115-4959-4913-8e7b-dd70d7651f07 in service fd00:1122:3344:101::2a + crucible 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service fd00:1122:3344:101::2c + crucible 8f3a1cc5-9195-4a30-ad02-b804278fe639 in service fd00:1122:3344:101::26 + crucible a1696cd4-588c-484a-b95b-66e824c0ce05 in service fd00:1122:3344:101::23 + crucible a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service fd00:1122:3344:101::28 + internal_dns 10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service fd00:1122:3344:2::1 + internal_ntp c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service fd00:1122:3344:101::21 + nexus 3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service fd00:1122:3344:101::22 sled 590e3034-d946-4166-b0e5-2d0034197a07: @@ -97,18 +99,19 @@ to: blueprint f432fcd5-1284-4058-8b4a-9286a3de6163 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 18f8fe40-646e-4962-b17a-20e201f3a6e5 in service fd00:1122:3344:102::2a - crucible 56d5d7cf-db2c-40a3-a775-003241ad4820 in service fd00:1122:3344:102::29 - crucible 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service fd00:1122:3344:102::2b - crucible 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service fd00:1122:3344:102::26 - crucible 93f2f40c-5616-4d8d-8519-ec6debdcede0 in service fd00:1122:3344:102::2c - crucible ab7ba6df-d401-40bd-940e-faf57c57aa2a in service fd00:1122:3344:102::28 - crucible af322036-371f-437c-8c08-7f40f3f1403b in service fd00:1122:3344:102::23 - crucible d637264f-6f40-44c2-8b7e-a179430210d2 in service fd00:1122:3344:102::25 - crucible dce226c9-7373-4bfa-8a94-79dc472857a6 in service fd00:1122:3344:102::27 - crucible edabedf3-839c-488d-ad6f-508ffa864674 in service fd00:1122:3344:102::24 - internal_ntp 47199d48-534c-4267-a654-d2d90e64b498 in service fd00:1122:3344:102::21 - nexus 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service fd00:1122:3344:102::22 + crucible 0565e7e4-f13a-4123-8928-d715f83e36aa in service fd00:1122:3344:102::2b + crucible 18f8fe40-646e-4962-b17a-20e201f3a6e5 in service fd00:1122:3344:102::27 + crucible 1cc3f503-2001-4d85-80e5-c7c40d2e3b10 in service fd00:1122:3344:102::2a + crucible 56d5d7cf-db2c-40a3-a775-003241ad4820 in service fd00:1122:3344:102::26 + crucible 62058f4c-c747-4e21-a8dc-2fd4a160c98c in service fd00:1122:3344:102::2c + crucible 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service fd00:1122:3344:102::28 + crucible 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service fd00:1122:3344:102::23 + crucible 93f2f40c-5616-4d8d-8519-ec6debdcede0 in service fd00:1122:3344:102::29 + crucible ab7ba6df-d401-40bd-940e-faf57c57aa2a in service fd00:1122:3344:102::25 + crucible dce226c9-7373-4bfa-8a94-79dc472857a6 in service fd00:1122:3344:102::24 + internal_dns d637264f-6f40-44c2-8b7e-a179430210d2 in service fd00:1122:3344:3::1 + internal_ntp af322036-371f-437c-8c08-7f40f3f1403b in service fd00:1122:3344:102::21 + nexus edabedf3-839c-488d-ad6f-508ffa864674 in service fd00:1122:3344:102::22 MODIFIED SLEDS: diff --git a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_1_2.txt b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_1_2.txt index 556ca094e1..f114e34241 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_1_2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_1_2.txt @@ -1,44 +1,6 @@ from: blueprint 516e80a3-b362-4fac-bd3c-4559717120dd to: blueprint 1ac2d88f-27dd-4506-8585-6b2be832528e - UNCHANGED SLEDS: - - sled d67ce8f0-a691-4010-b414-420d82e80527: - - physical disks at generation 1: - ---------------------------------------------------------------------- - vendor model serial - ---------------------------------------------------------------------- - fake-vendor fake-model serial-1e2ec79e-9c11-4133-ac77-e0b994a507d5 - fake-vendor fake-model serial-440ae69d-5e2e-4539-91d0-e2930bdd7203 - fake-vendor fake-model serial-4e91d4a3-bb6c-44bb-bd4e-bf8913c1ba2b - fake-vendor fake-model serial-67de3a80-29cb-4066-b743-e285a2ca1f4e - fake-vendor fake-model serial-9139b70f-c1d3-475d-8f02-7c9acba52b2b - fake-vendor fake-model serial-95fbb110-5272-4646-ab50-21b31b7cde23 - fake-vendor fake-model serial-9bf35cd7-4938-4c34-8189-288b3195cb64 - fake-vendor fake-model serial-9d833141-18a1-4f24-8a34-6076c026aa87 - fake-vendor fake-model serial-a279461f-a7b9-413f-a79f-cb4dab4c3fce - fake-vendor fake-model serial-ff7e002b-3ad8-4d45-b03a-c46ef0ac8e59 - - - omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 15dbaa30-1539-49d6-970d-ba5962960f33 in service fd00:1122:3344:101::27 - crucible 1ec4cc7b-2f00-4d13-8176-3b9815533ae9 in service fd00:1122:3344:101::24 - crucible 2e65b765-5c41-4519-bf4e-e2a68569afc1 in service fd00:1122:3344:101::23 - crucible 3d4143df-e212-4774-9258-7d9b421fac2e in service fd00:1122:3344:101::25 - crucible 5d9d8fa7-8379-470b-90ba-fe84a3c45512 in service fd00:1122:3344:101::2a - crucible 70232a6d-6c9d-4fa6-a34d-9c73d940db33 in service fd00:1122:3344:101::28 - crucible 8567a616-a709-4c8c-a323-4474675dad5c in service fd00:1122:3344:101::2c - crucible 8b0b8623-930a-41af-9f9b-ca28b1b11139 in service fd00:1122:3344:101::29 - crucible cf87d2a3-d323-44a3-a87e-adc4ef6c75f4 in service fd00:1122:3344:101::2b - crucible eac6c0a0-baa5-4490-9cee-65198b7fbd9c in service fd00:1122:3344:101::26 - internal_ntp ad76d200-5675-444b-b19c-684689ff421f in service fd00:1122:3344:101::21 - nexus e9bf2525-5fa0-4c1b-b52d-481225083845 in service fd00:1122:3344:101::22 - - MODIFIED SLEDS: sled a1b477db-b629-48eb-911d-1ccdafca75b9: @@ -63,25 +25,27 @@ to: blueprint 1ac2d88f-27dd-4506-8585-6b2be832528e ------------------------------------------------------------------------------------------- zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------- -* crucible 1e1ed0cc-1adc-410f-943a-d1a3107de619 - in service fd00:1122:3344:103::27 +* crucible 1e1ed0cc-1adc-410f-943a-d1a3107de619 - in service fd00:1122:3344:103::26 + └─ + expunged +* crucible 2307bbed-02ba-493b-89e3-46585c74c8fc - in service fd00:1122:3344:103::27 └─ + expunged -* crucible 2307bbed-02ba-493b-89e3-46585c74c8fc - in service fd00:1122:3344:103::28 +* crucible 603e629d-2599-400e-b879-4134d4cc426e - in service fd00:1122:3344:103::2b └─ + expunged -* crucible 4e36b7ef-5684-4304-b7c3-3c31aaf83d4f - in service fd00:1122:3344:103::23 +* crucible 9179d6dc-387d-424e-8d62-ed59b2c728f6 - in service fd00:1122:3344:103::29 └─ + expunged -* crucible 603e629d-2599-400e-b879-4134d4cc426e - in service fd00:1122:3344:103::2c +* crucible ad76d200-5675-444b-b19c-684689ff421f - in service fd00:1122:3344:103::2c └─ + expunged -* crucible 9179d6dc-387d-424e-8d62-ed59b2c728f6 - in service fd00:1122:3344:103::2a +* crucible c28d7b4b-a259-45ad-945d-f19ca3c6964c - in service fd00:1122:3344:103::28 └─ + expunged -* crucible c28d7b4b-a259-45ad-945d-f19ca3c6964c - in service fd00:1122:3344:103::29 +* crucible e29998e7-9ed2-46b6-bb70-4118159fe07f - in service fd00:1122:3344:103::25 └─ + expunged -* crucible e29998e7-9ed2-46b6-bb70-4118159fe07f - in service fd00:1122:3344:103::26 +* crucible f06e91a1-0c17-4cca-adbc-1c9b67bdb11d - in service fd00:1122:3344:103::2a └─ + expunged -* crucible f06e91a1-0c17-4cca-adbc-1c9b67bdb11d - in service fd00:1122:3344:103::2b +* crucible f11f5c60-1ac7-4630-9a3a-a9bc85c75203 - in service fd00:1122:3344:103::24 └─ + expunged -* crucible f11f5c60-1ac7-4630-9a3a-a9bc85c75203 - in service fd00:1122:3344:103::25 +* crucible f231e4eb-3fc9-4964-9d71-2c41644852d9 - in service fd00:1122:3344:103::23 └─ + expunged -* crucible f231e4eb-3fc9-4964-9d71-2c41644852d9 - in service fd00:1122:3344:103::24 +* internal_dns 4e36b7ef-5684-4304-b7c3-3c31aaf83d4f - in service fd00:1122:3344:1::1 └─ + expunged * internal_ntp c62b87b6-b98d-4d22-ba4f-cee4499e2ba8 - in service fd00:1122:3344:103::21 └─ + expunged @@ -89,6 +53,44 @@ to: blueprint 1ac2d88f-27dd-4506-8585-6b2be832528e └─ + expunged + sled d67ce8f0-a691-4010-b414-420d82e80527: + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-1e2ec79e-9c11-4133-ac77-e0b994a507d5 + fake-vendor fake-model serial-440ae69d-5e2e-4539-91d0-e2930bdd7203 + fake-vendor fake-model serial-4e91d4a3-bb6c-44bb-bd4e-bf8913c1ba2b + fake-vendor fake-model serial-67de3a80-29cb-4066-b743-e285a2ca1f4e + fake-vendor fake-model serial-9139b70f-c1d3-475d-8f02-7c9acba52b2b + fake-vendor fake-model serial-95fbb110-5272-4646-ab50-21b31b7cde23 + fake-vendor fake-model serial-9bf35cd7-4938-4c34-8189-288b3195cb64 + fake-vendor fake-model serial-9d833141-18a1-4f24-8a34-6076c026aa87 + fake-vendor fake-model serial-a279461f-a7b9-413f-a79f-cb4dab4c3fce + fake-vendor fake-model serial-ff7e002b-3ad8-4d45-b03a-c46ef0ac8e59 + + + omicron zones generation 2 -> 3: + ------------------------------------------------------------------------------------------ + zone type zone id disposition underlay IP + ------------------------------------------------------------------------------------------ + crucible 15dbaa30-1539-49d6-970d-ba5962960f33 in service fd00:1122:3344:101::25 + crucible 3d4143df-e212-4774-9258-7d9b421fac2e in service fd00:1122:3344:101::23 + crucible 5d9d8fa7-8379-470b-90ba-fe84a3c45512 in service fd00:1122:3344:101::28 + crucible 70232a6d-6c9d-4fa6-a34d-9c73d940db33 in service fd00:1122:3344:101::26 + crucible 8567a616-a709-4c8c-a323-4474675dad5c in service fd00:1122:3344:101::2a + crucible 8b0b8623-930a-41af-9f9b-ca28b1b11139 in service fd00:1122:3344:101::27 + crucible 99c6401d-9796-4ae1-bf0c-9a097cf21c33 in service fd00:1122:3344:101::2c + crucible cf87d2a3-d323-44a3-a87e-adc4ef6c75f4 in service fd00:1122:3344:101::29 + crucible eac6c0a0-baa5-4490-9cee-65198b7fbd9c in service fd00:1122:3344:101::24 + crucible f68846ad-4619-4747-8293-a2b4aeeafc5b in service fd00:1122:3344:101::2b + internal_dns 1ec4cc7b-2f00-4d13-8176-3b9815533ae9 in service fd00:1122:3344:2::1 + internal_ntp e9bf2525-5fa0-4c1b-b52d-481225083845 in service fd00:1122:3344:101::21 + nexus 2e65b765-5c41-4519-bf4e-e2a68569afc1 in service fd00:1122:3344:101::22 ++ nexus ff9ce09c-afbf-425b-bbfa-3d8fb254f98e in service fd00:1122:3344:101::2d + + sled fefcf4cf-f7e7-46b3-b629-058526ce440e: physical disks at generation 1: @@ -111,19 +113,20 @@ to: blueprint 1ac2d88f-27dd-4506-8585-6b2be832528e ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 0e2b035e-1de1-48af-8ac0-5316418e3de1 in service fd00:1122:3344:102::2a - crucible 4f8ce495-21dd-48a1-859c-80d34ce394ed in service fd00:1122:3344:102::23 - crucible 5c78756d-6182-4c27-a507-3419e8dbe76b in service fd00:1122:3344:102::28 - crucible a1ae92ac-e1f1-4654-ab54-5b75ba7c44d6 in service fd00:1122:3344:102::24 - crucible a308d3e1-118c-440a-947a-8b6ab7d833ab in service fd00:1122:3344:102::25 - crucible b7402110-d88f-4ca4-8391-4a2fda6ad271 in service fd00:1122:3344:102::29 - crucible b7ae596e-0c85-40b2-bb47-df9f76db3cca in service fd00:1122:3344:102::2b - crucible c552280f-ba02-4f8d-9049-bd269e6b7845 in service fd00:1122:3344:102::26 - crucible cf13b878-47f1-4ba0-b8c2-9f3e15f2ee87 in service fd00:1122:3344:102::2c - crucible e6d0df1f-9f98-4c5a-9540-8444d1185c7d in service fd00:1122:3344:102::27 - internal_ntp f68846ad-4619-4747-8293-a2b4aeeafc5b in service fd00:1122:3344:102::21 - nexus 99c6401d-9796-4ae1-bf0c-9a097cf21c33 in service fd00:1122:3344:102::22 -+ nexus c8851a11-a4f7-4b21-9281-6182fd15dc8d in service fd00:1122:3344:102::2d + crucible 0e2b035e-1de1-48af-8ac0-5316418e3de1 in service fd00:1122:3344:102::27 + crucible 2bf9ee97-90e1-48a7-bb06-a35cec63b7fe in service fd00:1122:3344:102::2b + crucible 5c78756d-6182-4c27-a507-3419e8dbe76b in service fd00:1122:3344:102::25 + crucible b7402110-d88f-4ca4-8391-4a2fda6ad271 in service fd00:1122:3344:102::26 + crucible b7ae596e-0c85-40b2-bb47-df9f76db3cca in service fd00:1122:3344:102::28 + crucible c552280f-ba02-4f8d-9049-bd269e6b7845 in service fd00:1122:3344:102::23 + crucible cf13b878-47f1-4ba0-b8c2-9f3e15f2ee87 in service fd00:1122:3344:102::29 + crucible e3bfcb1e-3708-45e7-a45a-2a2cab7ad829 in service fd00:1122:3344:102::2c + crucible e6d0df1f-9f98-4c5a-9540-8444d1185c7d in service fd00:1122:3344:102::24 + crucible eb034526-1767-4cc4-8225-ec962265710b in service fd00:1122:3344:102::2a + internal_dns a308d3e1-118c-440a-947a-8b6ab7d833ab in service fd00:1122:3344:3::1 + internal_ntp 4f8ce495-21dd-48a1-859c-80d34ce394ed in service fd00:1122:3344:102::21 + nexus a1ae92ac-e1f1-4654-ab54-5b75ba7c44d6 in service fd00:1122:3344:102::22 ++ internal_dns c8851a11-a4f7-4b21-9281-6182fd15dc8d in service fd00:1122:3344:4::1 COCKROACHDB SETTINGS: diff --git a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt index 6954d4e12b..5e48bdd646 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt @@ -19,22 +19,24 @@ parent: 516e80a3-b362-4fac-bd3c-4559717120dd fake-vendor fake-model serial-ff7e002b-3ad8-4d45-b03a-c46ef0ac8e59 - omicron zones at generation 2: + omicron zones at generation 3: ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 15dbaa30-1539-49d6-970d-ba5962960f33 in service fd00:1122:3344:101::27 - crucible 1ec4cc7b-2f00-4d13-8176-3b9815533ae9 in service fd00:1122:3344:101::24 - crucible 2e65b765-5c41-4519-bf4e-e2a68569afc1 in service fd00:1122:3344:101::23 - crucible 3d4143df-e212-4774-9258-7d9b421fac2e in service fd00:1122:3344:101::25 - crucible 5d9d8fa7-8379-470b-90ba-fe84a3c45512 in service fd00:1122:3344:101::2a - crucible 70232a6d-6c9d-4fa6-a34d-9c73d940db33 in service fd00:1122:3344:101::28 - crucible 8567a616-a709-4c8c-a323-4474675dad5c in service fd00:1122:3344:101::2c - crucible 8b0b8623-930a-41af-9f9b-ca28b1b11139 in service fd00:1122:3344:101::29 - crucible cf87d2a3-d323-44a3-a87e-adc4ef6c75f4 in service fd00:1122:3344:101::2b - crucible eac6c0a0-baa5-4490-9cee-65198b7fbd9c in service fd00:1122:3344:101::26 - internal_ntp ad76d200-5675-444b-b19c-684689ff421f in service fd00:1122:3344:101::21 - nexus e9bf2525-5fa0-4c1b-b52d-481225083845 in service fd00:1122:3344:101::22 + crucible 15dbaa30-1539-49d6-970d-ba5962960f33 in service fd00:1122:3344:101::25 + crucible 3d4143df-e212-4774-9258-7d9b421fac2e in service fd00:1122:3344:101::23 + crucible 5d9d8fa7-8379-470b-90ba-fe84a3c45512 in service fd00:1122:3344:101::28 + crucible 70232a6d-6c9d-4fa6-a34d-9c73d940db33 in service fd00:1122:3344:101::26 + crucible 8567a616-a709-4c8c-a323-4474675dad5c in service fd00:1122:3344:101::2a + crucible 8b0b8623-930a-41af-9f9b-ca28b1b11139 in service fd00:1122:3344:101::27 + crucible 99c6401d-9796-4ae1-bf0c-9a097cf21c33 in service fd00:1122:3344:101::2c + crucible cf87d2a3-d323-44a3-a87e-adc4ef6c75f4 in service fd00:1122:3344:101::29 + crucible eac6c0a0-baa5-4490-9cee-65198b7fbd9c in service fd00:1122:3344:101::24 + crucible f68846ad-4619-4747-8293-a2b4aeeafc5b in service fd00:1122:3344:101::2b + internal_dns 1ec4cc7b-2f00-4d13-8176-3b9815533ae9 in service fd00:1122:3344:2::1 + internal_ntp e9bf2525-5fa0-4c1b-b52d-481225083845 in service fd00:1122:3344:101::21 + nexus 2e65b765-5c41-4519-bf4e-e2a68569afc1 in service fd00:1122:3344:101::22 + nexus ff9ce09c-afbf-425b-bbfa-3d8fb254f98e in service fd00:1122:3344:101::2d @@ -60,19 +62,20 @@ parent: 516e80a3-b362-4fac-bd3c-4559717120dd ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 0e2b035e-1de1-48af-8ac0-5316418e3de1 in service fd00:1122:3344:102::2a - crucible 4f8ce495-21dd-48a1-859c-80d34ce394ed in service fd00:1122:3344:102::23 - crucible 5c78756d-6182-4c27-a507-3419e8dbe76b in service fd00:1122:3344:102::28 - crucible a1ae92ac-e1f1-4654-ab54-5b75ba7c44d6 in service fd00:1122:3344:102::24 - crucible a308d3e1-118c-440a-947a-8b6ab7d833ab in service fd00:1122:3344:102::25 - crucible b7402110-d88f-4ca4-8391-4a2fda6ad271 in service fd00:1122:3344:102::29 - crucible b7ae596e-0c85-40b2-bb47-df9f76db3cca in service fd00:1122:3344:102::2b - crucible c552280f-ba02-4f8d-9049-bd269e6b7845 in service fd00:1122:3344:102::26 - crucible cf13b878-47f1-4ba0-b8c2-9f3e15f2ee87 in service fd00:1122:3344:102::2c - crucible e6d0df1f-9f98-4c5a-9540-8444d1185c7d in service fd00:1122:3344:102::27 - internal_ntp f68846ad-4619-4747-8293-a2b4aeeafc5b in service fd00:1122:3344:102::21 - nexus 99c6401d-9796-4ae1-bf0c-9a097cf21c33 in service fd00:1122:3344:102::22 - nexus c8851a11-a4f7-4b21-9281-6182fd15dc8d in service fd00:1122:3344:102::2d + crucible 0e2b035e-1de1-48af-8ac0-5316418e3de1 in service fd00:1122:3344:102::27 + crucible 2bf9ee97-90e1-48a7-bb06-a35cec63b7fe in service fd00:1122:3344:102::2b + crucible 5c78756d-6182-4c27-a507-3419e8dbe76b in service fd00:1122:3344:102::25 + crucible b7402110-d88f-4ca4-8391-4a2fda6ad271 in service fd00:1122:3344:102::26 + crucible b7ae596e-0c85-40b2-bb47-df9f76db3cca in service fd00:1122:3344:102::28 + crucible c552280f-ba02-4f8d-9049-bd269e6b7845 in service fd00:1122:3344:102::23 + crucible cf13b878-47f1-4ba0-b8c2-9f3e15f2ee87 in service fd00:1122:3344:102::29 + crucible e3bfcb1e-3708-45e7-a45a-2a2cab7ad829 in service fd00:1122:3344:102::2c + crucible e6d0df1f-9f98-4c5a-9540-8444d1185c7d in service fd00:1122:3344:102::24 + crucible eb034526-1767-4cc4-8225-ec962265710b in service fd00:1122:3344:102::2a + internal_dns a308d3e1-118c-440a-947a-8b6ab7d833ab in service fd00:1122:3344:3::1 + internal_dns c8851a11-a4f7-4b21-9281-6182fd15dc8d in service fd00:1122:3344:4::1 + internal_ntp 4f8ce495-21dd-48a1-859c-80d34ce394ed in service fd00:1122:3344:102::21 + nexus a1ae92ac-e1f1-4654-ab54-5b75ba7c44d6 in service fd00:1122:3344:102::22 @@ -82,16 +85,17 @@ WARNING: Zones exist without physical disks! ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 1e1ed0cc-1adc-410f-943a-d1a3107de619 expunged fd00:1122:3344:103::27 - crucible 2307bbed-02ba-493b-89e3-46585c74c8fc expunged fd00:1122:3344:103::28 - crucible 4e36b7ef-5684-4304-b7c3-3c31aaf83d4f expunged fd00:1122:3344:103::23 - crucible 603e629d-2599-400e-b879-4134d4cc426e expunged fd00:1122:3344:103::2c - crucible 9179d6dc-387d-424e-8d62-ed59b2c728f6 expunged fd00:1122:3344:103::2a - crucible c28d7b4b-a259-45ad-945d-f19ca3c6964c expunged fd00:1122:3344:103::29 - crucible e29998e7-9ed2-46b6-bb70-4118159fe07f expunged fd00:1122:3344:103::26 - crucible f06e91a1-0c17-4cca-adbc-1c9b67bdb11d expunged fd00:1122:3344:103::2b - crucible f11f5c60-1ac7-4630-9a3a-a9bc85c75203 expunged fd00:1122:3344:103::25 - crucible f231e4eb-3fc9-4964-9d71-2c41644852d9 expunged fd00:1122:3344:103::24 + crucible 1e1ed0cc-1adc-410f-943a-d1a3107de619 expunged fd00:1122:3344:103::26 + crucible 2307bbed-02ba-493b-89e3-46585c74c8fc expunged fd00:1122:3344:103::27 + crucible 603e629d-2599-400e-b879-4134d4cc426e expunged fd00:1122:3344:103::2b + crucible 9179d6dc-387d-424e-8d62-ed59b2c728f6 expunged fd00:1122:3344:103::29 + crucible ad76d200-5675-444b-b19c-684689ff421f expunged fd00:1122:3344:103::2c + crucible c28d7b4b-a259-45ad-945d-f19ca3c6964c expunged fd00:1122:3344:103::28 + crucible e29998e7-9ed2-46b6-bb70-4118159fe07f expunged fd00:1122:3344:103::25 + crucible f06e91a1-0c17-4cca-adbc-1c9b67bdb11d expunged fd00:1122:3344:103::2a + crucible f11f5c60-1ac7-4630-9a3a-a9bc85c75203 expunged fd00:1122:3344:103::24 + crucible f231e4eb-3fc9-4964-9d71-2c41644852d9 expunged fd00:1122:3344:103::23 + internal_dns 4e36b7ef-5684-4304-b7c3-3c31aaf83d4f expunged fd00:1122:3344:1::1 internal_ntp c62b87b6-b98d-4d22-ba4f-cee4499e2ba8 expunged fd00:1122:3344:103::21 nexus 6a70a233-1900-43c0-9c00-aa9d1f7adfbc expunged fd00:1122:3344:103::22 @@ -104,7 +108,7 @@ WARNING: Zones exist without physical disks! METADATA: created by::::::::::: test_blueprint2 created at::::::::::: 1970-01-01T00:00:00.000Z - comment:::::::::::::: sled a1b477db-b629-48eb-911d-1ccdafca75b9: expunged 12 zones because: sled policy is expunged + comment:::::::::::::: sled a1b477db-b629-48eb-911d-1ccdafca75b9: expunged 13 zones because: sled policy is expunged internal DNS version: 1 external DNS version: 1 diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt index d3f667170c..2199ce79e7 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt @@ -25,16 +25,17 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be in service fd00:1122:3344:105::26 - crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2c - crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::27 - crucible 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:105::23 - crucible 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::25 - crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::28 - crucible c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:105::24 - crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::2a - crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2b - crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::29 + crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be in service fd00:1122:3344:105::25 + crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2b + crucible 67d913e0-0005-4599-9b28-0abbf6cc2916 in service fd00:1122:3344:105::2c + crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::26 + crucible 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::24 + crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::27 + crucible c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:105::23 + crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::29 + crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2a + crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::28 + internal_dns 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:1::1 internal_ntp 7f4e9f9f-08f8-4d14-885d-e977c05525ad in service fd00:1122:3344:105::21 nexus 6dff7633-66bb-4924-a6ff-2c896e66964b in service fd00:1122:3344:105::22 @@ -63,29 +64,31 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------- zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------- -* crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 - in service fd00:1122:3344:103::2c +* crucible 01d58626-e1b0-480f-96be-ac784863c7dc - in service fd00:1122:3344:103::2c └─ + expunged -* crucible 0dcfdfc5-481e-4153-b97c-11cf02b648ea - in service fd00:1122:3344:103::25 +* crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 - in service fd00:1122:3344:103::2a └─ + expunged -* crucible 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f - in service fd00:1122:3344:103::27 +* crucible 0dcfdfc5-481e-4153-b97c-11cf02b648ea - in service fd00:1122:3344:103::23 └─ + expunged -* crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 - in service fd00:1122:3344:103::28 +* crucible 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f - in service fd00:1122:3344:103::25 └─ + expunged -* crucible 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb - in service fd00:1122:3344:103::24 +* crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 - in service fd00:1122:3344:103::26 └─ + expunged -* crucible 67622d61-2df4-414d-aa0e-d1277265f405 - in service fd00:1122:3344:103::23 +* crucible b91b271d-8d80-4f49-99a0-34006ae86063 - in service fd00:1122:3344:103::28 └─ + expunged -* crucible b91b271d-8d80-4f49-99a0-34006ae86063 - in service fd00:1122:3344:103::2a +* crucible d6ee1338-3127-43ec-9aaa-b973ccf05496 - in service fd00:1122:3344:103::24 └─ + expunged -* crucible d6ee1338-3127-43ec-9aaa-b973ccf05496 - in service fd00:1122:3344:103::26 +* crucible e39d7c9e-182b-48af-af87-58079d723583 - in service fd00:1122:3344:103::27 └─ + expunged -* crucible e39d7c9e-182b-48af-af87-58079d723583 - in service fd00:1122:3344:103::29 +* crucible f3f2e4f3-0985-4ef6-8336-ce479382d05d - in service fd00:1122:3344:103::2b └─ + expunged -* crucible f69f92a1-5007-4bb0-a85b-604dc217154b - in service fd00:1122:3344:103::2b +* crucible f69f92a1-5007-4bb0-a85b-604dc217154b - in service fd00:1122:3344:103::29 └─ + expunged -* internal_ntp 67d913e0-0005-4599-9b28-0abbf6cc2916 - in service fd00:1122:3344:103::21 +* internal_dns 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb - in service fd00:1122:3344:2::1 └─ + expunged -* nexus 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb - in service fd00:1122:3344:103::22 +* internal_ntp 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb - in service fd00:1122:3344:103::21 + └─ + expunged +* nexus 67622d61-2df4-414d-aa0e-d1277265f405 - in service fd00:1122:3344:103::22 └─ + expunged @@ -111,18 +114,19 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::2c - crucible 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:102::23 - crucible 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:102::24 - crucible 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::29 - crucible 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:102::25 - crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::2b - crucible 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::26 - crucible b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::27 - crucible b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:102::28 - crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::2a - internal_ntp f3f2e4f3-0985-4ef6-8336-ce479382d05d expunged fd00:1122:3344:102::21 - nexus 01d58626-e1b0-480f-96be-ac784863c7dc expunged fd00:1122:3344:102::22 + crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::29 + crucible 57b96d5c-b71e-43e4-8869-7d514003d00d expunged fd00:1122:3344:102::2a + crucible 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::26 + crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::28 + crucible 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::23 + crucible b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::24 + crucible b4947d31-f70e-4ee0-8817-0ca6cea9b16b expunged fd00:1122:3344:102::2b + crucible b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:102::25 + crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::27 + crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a expunged fd00:1122:3344:102::2c + internal_dns 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:3::1 + internal_ntp 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:102::21 + nexus 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:102::22 sled 75bc286f-2b4b-482c-9431-59272af529da: @@ -147,18 +151,19 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 15bb9def-69b8-4d2e-b04f-9fee1143387c in service fd00:1122:3344:104::25 - crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::2c - crucible 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::28 - crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::2a - crucible 996d7570-b0df-46d5-aaa4-0c97697cf484 in service fd00:1122:3344:104::26 - crucible a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::29 - crucible b1783e95-9598-451d-b6ba-c50b52b428c3 in service fd00:1122:3344:104::24 - crucible c6dd531e-2d1d-423b-acc8-358533dab78c in service fd00:1122:3344:104::27 - crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a in service fd00:1122:3344:104::23 - crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::2b - internal_ntp 57b96d5c-b71e-43e4-8869-7d514003d00d in service fd00:1122:3344:104::21 - nexus b4947d31-f70e-4ee0-8817-0ca6cea9b16b in service fd00:1122:3344:104::22 + crucible 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:104::2a + crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::28 + crucible 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:104::2c + crucible 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::24 + crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::26 + crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:104::2b + crucible a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::25 + crucible c6dd531e-2d1d-423b-acc8-358533dab78c in service fd00:1122:3344:104::23 + crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::27 + crucible f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:104::29 + internal_dns 996d7570-b0df-46d5-aaa4-0c97697cf484 in service fd00:1122:3344:4::1 + internal_ntp b1783e95-9598-451d-b6ba-c50b52b428c3 in service fd00:1122:3344:104::21 + nexus 15bb9def-69b8-4d2e-b04f-9fee1143387c in service fd00:1122:3344:104::22 + nexus 2ec75441-3d7d-4b4b-9614-af03de5a3666 in service fd00:1122:3344:104::2d + nexus 508abd03-cbfe-4654-9a6d-7f15a1ad32e5 in service fd00:1122:3344:104::2e + nexus 59950bc8-1497-44dd-8cbf-b6502ba921b2 in service fd00:1122:3344:104::2f @@ -186,18 +191,19 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:101::27 - crucible 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:101::24 - crucible 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::29 - crucible 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:101::26 - crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:101::23 - crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::2a - crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::2c - crucible bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::28 - crucible c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:101::25 - crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::2b - internal_ntp f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:101::21 - nexus 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:101::22 + crucible 414830dc-c8c1-4748-9e9e-bc3a6435a93c in service fd00:1122:3344:101::2b + crucible 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::24 + crucible 772cbcbd-58be-4158-be85-be744871fa22 in service fd00:1122:3344:101::28 + crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::25 + crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::27 + crucible be75764a-491b-4aec-992e-1c39e25de975 in service fd00:1122:3344:101::29 + crucible be920398-024a-4655-8c49-69b5ac48dfff in service fd00:1122:3344:101::2c + crucible bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::23 + crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::26 + crucible e001fea0-6594-4ece-97e3-6198c293e931 in service fd00:1122:3344:101::2a + internal_dns 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:5::1 + internal_ntp c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:101::21 + nexus 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:101::22 + nexus 3ca5292f-8a59-4475-bb72-0f43714d0fff in service fd00:1122:3344:101::2e + nexus 99f6d544-8599-4e2b-a55a-82d9e0034662 in service fd00:1122:3344:101::2d + nexus c26b3bda-5561-44a1-a69f-22103fe209a1 in service fd00:1122:3344:101::2f diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt index 837cc56553..7ff2d585e1 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt @@ -25,21 +25,22 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 15bb9def-69b8-4d2e-b04f-9fee1143387c in service fd00:1122:3344:104::25 - crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::2c - crucible 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::28 - crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::2a - crucible 996d7570-b0df-46d5-aaa4-0c97697cf484 in service fd00:1122:3344:104::26 - crucible a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::29 - crucible b1783e95-9598-451d-b6ba-c50b52b428c3 in service fd00:1122:3344:104::24 - crucible c6dd531e-2d1d-423b-acc8-358533dab78c in service fd00:1122:3344:104::27 - crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a in service fd00:1122:3344:104::23 - crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::2b - internal_ntp 57b96d5c-b71e-43e4-8869-7d514003d00d in service fd00:1122:3344:104::21 + crucible 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:104::2a + crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::28 + crucible 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:104::2c + crucible 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::24 + crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::26 + crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:104::2b + crucible a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::25 + crucible c6dd531e-2d1d-423b-acc8-358533dab78c in service fd00:1122:3344:104::23 + crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::27 + crucible f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:104::29 + internal_dns 996d7570-b0df-46d5-aaa4-0c97697cf484 in service fd00:1122:3344:4::1 + internal_ntp b1783e95-9598-451d-b6ba-c50b52b428c3 in service fd00:1122:3344:104::21 + nexus 15bb9def-69b8-4d2e-b04f-9fee1143387c in service fd00:1122:3344:104::22 nexus 2ec75441-3d7d-4b4b-9614-af03de5a3666 in service fd00:1122:3344:104::2d nexus 508abd03-cbfe-4654-9a6d-7f15a1ad32e5 in service fd00:1122:3344:104::2e nexus 59950bc8-1497-44dd-8cbf-b6502ba921b2 in service fd00:1122:3344:104::2f - nexus b4947d31-f70e-4ee0-8817-0ca6cea9b16b in service fd00:1122:3344:104::22 sled affab35f-600a-4109-8ea0-34a067a4e0bc: @@ -64,19 +65,20 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:101::27 - crucible 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:101::24 - crucible 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::29 - crucible 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:101::26 - crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:101::23 - crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::2a - crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::2c - crucible bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::28 - crucible c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:101::25 - crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::2b - internal_ntp f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:101::21 - nexus 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:101::22 + crucible 414830dc-c8c1-4748-9e9e-bc3a6435a93c in service fd00:1122:3344:101::2b + crucible 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::24 + crucible 772cbcbd-58be-4158-be85-be744871fa22 in service fd00:1122:3344:101::28 + crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::25 + crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::27 + crucible be75764a-491b-4aec-992e-1c39e25de975 in service fd00:1122:3344:101::29 + crucible be920398-024a-4655-8c49-69b5ac48dfff in service fd00:1122:3344:101::2c + crucible bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::23 + crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::26 + crucible e001fea0-6594-4ece-97e3-6198c293e931 in service fd00:1122:3344:101::2a + internal_dns 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:5::1 + internal_ntp c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:101::21 nexus 3ca5292f-8a59-4475-bb72-0f43714d0fff in service fd00:1122:3344:101::2e + nexus 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:101::22 nexus 99f6d544-8599-4e2b-a55a-82d9e0034662 in service fd00:1122:3344:101::2d nexus c26b3bda-5561-44a1-a69f-22103fe209a1 in service fd00:1122:3344:101::2f @@ -89,18 +91,19 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ -- crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::2c -- crucible 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:102::23 -- crucible 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:102::24 -- crucible 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::29 -- crucible 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:102::25 -- crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::2b -- crucible 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::26 -- crucible b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::27 -- crucible b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:102::28 -- crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::2a -- internal_ntp f3f2e4f3-0985-4ef6-8336-ce479382d05d expunged fd00:1122:3344:102::21 -- nexus 01d58626-e1b0-480f-96be-ac784863c7dc expunged fd00:1122:3344:102::22 +- crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::29 +- crucible 57b96d5c-b71e-43e4-8869-7d514003d00d expunged fd00:1122:3344:102::2a +- crucible 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::26 +- crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::28 +- crucible 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::23 +- crucible b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::24 +- crucible b4947d31-f70e-4ee0-8817-0ca6cea9b16b expunged fd00:1122:3344:102::2b +- crucible b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:102::25 +- crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::27 +- crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a expunged fd00:1122:3344:102::2c +- internal_dns 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:3::1 +- internal_ntp 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:102::21 +- nexus 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:102::22 MODIFIED SLEDS: @@ -124,20 +127,21 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 omicron zones at generation 2: - ---------------------------------------------------------------------------------------- - zone type zone id disposition underlay IP - ---------------------------------------------------------------------------------------- - crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::27 - crucible 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:105::23 - crucible 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::25 - crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::28 - crucible c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:105::24 - crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::2a - crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2b - crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::29 -- crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2c -* crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be - in service fd00:1122:3344:105::26 - └─ + quiesced + ------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + ------------------------------------------------------------------------------------------- + crucible 67d913e0-0005-4599-9b28-0abbf6cc2916 in service fd00:1122:3344:105::2c + crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::26 + crucible 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::24 + crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::27 + crucible c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:105::23 + crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::29 + crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2a + crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::28 + internal_dns 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:1::1 +- crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2b +* crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be - in service fd00:1122:3344:105::25 + └─ + quiesced sled 48d95fef-bc9f-4f50-9a53-1e075836291d: @@ -146,18 +150,19 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ -- crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 expunged fd00:1122:3344:103::2c -- crucible 0dcfdfc5-481e-4153-b97c-11cf02b648ea expunged fd00:1122:3344:103::25 -- crucible 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f expunged fd00:1122:3344:103::27 -- crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 expunged fd00:1122:3344:103::28 -- crucible 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb expunged fd00:1122:3344:103::24 -- crucible 67622d61-2df4-414d-aa0e-d1277265f405 expunged fd00:1122:3344:103::23 -- crucible b91b271d-8d80-4f49-99a0-34006ae86063 expunged fd00:1122:3344:103::2a -- crucible d6ee1338-3127-43ec-9aaa-b973ccf05496 expunged fd00:1122:3344:103::26 -- crucible e39d7c9e-182b-48af-af87-58079d723583 expunged fd00:1122:3344:103::29 -- crucible f69f92a1-5007-4bb0-a85b-604dc217154b expunged fd00:1122:3344:103::2b -- internal_ntp 67d913e0-0005-4599-9b28-0abbf6cc2916 expunged fd00:1122:3344:103::21 -- nexus 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb expunged fd00:1122:3344:103::22 +- crucible 01d58626-e1b0-480f-96be-ac784863c7dc expunged fd00:1122:3344:103::2c +- crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 expunged fd00:1122:3344:103::2a +- crucible 0dcfdfc5-481e-4153-b97c-11cf02b648ea expunged fd00:1122:3344:103::23 +- crucible 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f expunged fd00:1122:3344:103::25 +- crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 expunged fd00:1122:3344:103::26 +- crucible b91b271d-8d80-4f49-99a0-34006ae86063 expunged fd00:1122:3344:103::28 +- crucible d6ee1338-3127-43ec-9aaa-b973ccf05496 expunged fd00:1122:3344:103::24 +- crucible e39d7c9e-182b-48af-af87-58079d723583 expunged fd00:1122:3344:103::27 +- crucible f3f2e4f3-0985-4ef6-8336-ce479382d05d expunged fd00:1122:3344:103::2b +- crucible f69f92a1-5007-4bb0-a85b-604dc217154b expunged fd00:1122:3344:103::29 +- internal_dns 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb expunged fd00:1122:3344:2::1 +- internal_ntp 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb expunged fd00:1122:3344:103::21 +- nexus 67622d61-2df4-414d-aa0e-d1277265f405 expunged fd00:1122:3344:103::22 ERRORS: diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt index 5a2ed5a28a..1305438388 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt @@ -23,16 +23,17 @@ parent: 4d4e6c38-cd95-4c4e-8f45-6af4d686964b ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be in service fd00:1122:3344:105::26 - crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2c - crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::27 - crucible 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:105::23 - crucible 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::25 - crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::28 - crucible c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:105::24 - crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::2a - crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2b - crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::29 + crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be in service fd00:1122:3344:105::25 + crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2b + crucible 67d913e0-0005-4599-9b28-0abbf6cc2916 in service fd00:1122:3344:105::2c + crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::26 + crucible 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::24 + crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::27 + crucible c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:105::23 + crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::29 + crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2a + crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::28 + internal_dns 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:1::1 internal_ntp 7f4e9f9f-08f8-4d14-885d-e977c05525ad in service fd00:1122:3344:105::21 nexus 6dff7633-66bb-4924-a6ff-2c896e66964b in service fd00:1122:3344:105::22 @@ -60,21 +61,22 @@ parent: 4d4e6c38-cd95-4c4e-8f45-6af4d686964b ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 15bb9def-69b8-4d2e-b04f-9fee1143387c in service fd00:1122:3344:104::25 - crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::2c - crucible 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::28 - crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::2a - crucible 996d7570-b0df-46d5-aaa4-0c97697cf484 in service fd00:1122:3344:104::26 - crucible a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::29 - crucible b1783e95-9598-451d-b6ba-c50b52b428c3 in service fd00:1122:3344:104::24 - crucible c6dd531e-2d1d-423b-acc8-358533dab78c in service fd00:1122:3344:104::27 - crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a in service fd00:1122:3344:104::23 - crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::2b - internal_ntp 57b96d5c-b71e-43e4-8869-7d514003d00d in service fd00:1122:3344:104::21 + crucible 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:104::2a + crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::28 + crucible 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:104::2c + crucible 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::24 + crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::26 + crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:104::2b + crucible a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::25 + crucible c6dd531e-2d1d-423b-acc8-358533dab78c in service fd00:1122:3344:104::23 + crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::27 + crucible f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:104::29 + internal_dns 996d7570-b0df-46d5-aaa4-0c97697cf484 in service fd00:1122:3344:4::1 + internal_ntp b1783e95-9598-451d-b6ba-c50b52b428c3 in service fd00:1122:3344:104::21 + nexus 15bb9def-69b8-4d2e-b04f-9fee1143387c in service fd00:1122:3344:104::22 nexus 2ec75441-3d7d-4b4b-9614-af03de5a3666 in service fd00:1122:3344:104::2d nexus 508abd03-cbfe-4654-9a6d-7f15a1ad32e5 in service fd00:1122:3344:104::2e nexus 59950bc8-1497-44dd-8cbf-b6502ba921b2 in service fd00:1122:3344:104::2f - nexus b4947d31-f70e-4ee0-8817-0ca6cea9b16b in service fd00:1122:3344:104::22 @@ -100,19 +102,20 @@ parent: 4d4e6c38-cd95-4c4e-8f45-6af4d686964b ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:101::27 - crucible 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:101::24 - crucible 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::29 - crucible 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:101::26 - crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:101::23 - crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::2a - crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::2c - crucible bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::28 - crucible c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:101::25 - crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::2b - internal_ntp f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:101::21 - nexus 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:101::22 + crucible 414830dc-c8c1-4748-9e9e-bc3a6435a93c in service fd00:1122:3344:101::2b + crucible 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::24 + crucible 772cbcbd-58be-4158-be85-be744871fa22 in service fd00:1122:3344:101::28 + crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::25 + crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::27 + crucible be75764a-491b-4aec-992e-1c39e25de975 in service fd00:1122:3344:101::29 + crucible be920398-024a-4655-8c49-69b5ac48dfff in service fd00:1122:3344:101::2c + crucible bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::23 + crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::26 + crucible e001fea0-6594-4ece-97e3-6198c293e931 in service fd00:1122:3344:101::2a + internal_dns 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:5::1 + internal_ntp c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:101::21 nexus 3ca5292f-8a59-4475-bb72-0f43714d0fff in service fd00:1122:3344:101::2e + nexus 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:101::22 nexus 99f6d544-8599-4e2b-a55a-82d9e0034662 in service fd00:1122:3344:101::2d nexus c26b3bda-5561-44a1-a69f-22103fe209a1 in service fd00:1122:3344:101::2f @@ -124,18 +127,19 @@ WARNING: Zones exist without physical disks! ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 expunged fd00:1122:3344:103::2c - crucible 0dcfdfc5-481e-4153-b97c-11cf02b648ea expunged fd00:1122:3344:103::25 - crucible 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f expunged fd00:1122:3344:103::27 - crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 expunged fd00:1122:3344:103::28 - crucible 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb expunged fd00:1122:3344:103::24 - crucible 67622d61-2df4-414d-aa0e-d1277265f405 expunged fd00:1122:3344:103::23 - crucible b91b271d-8d80-4f49-99a0-34006ae86063 expunged fd00:1122:3344:103::2a - crucible d6ee1338-3127-43ec-9aaa-b973ccf05496 expunged fd00:1122:3344:103::26 - crucible e39d7c9e-182b-48af-af87-58079d723583 expunged fd00:1122:3344:103::29 - crucible f69f92a1-5007-4bb0-a85b-604dc217154b expunged fd00:1122:3344:103::2b - internal_ntp 67d913e0-0005-4599-9b28-0abbf6cc2916 expunged fd00:1122:3344:103::21 - nexus 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb expunged fd00:1122:3344:103::22 + crucible 01d58626-e1b0-480f-96be-ac784863c7dc expunged fd00:1122:3344:103::2c + crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 expunged fd00:1122:3344:103::2a + crucible 0dcfdfc5-481e-4153-b97c-11cf02b648ea expunged fd00:1122:3344:103::23 + crucible 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f expunged fd00:1122:3344:103::25 + crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 expunged fd00:1122:3344:103::26 + crucible b91b271d-8d80-4f49-99a0-34006ae86063 expunged fd00:1122:3344:103::28 + crucible d6ee1338-3127-43ec-9aaa-b973ccf05496 expunged fd00:1122:3344:103::24 + crucible e39d7c9e-182b-48af-af87-58079d723583 expunged fd00:1122:3344:103::27 + crucible f3f2e4f3-0985-4ef6-8336-ce479382d05d expunged fd00:1122:3344:103::2b + crucible f69f92a1-5007-4bb0-a85b-604dc217154b expunged fd00:1122:3344:103::29 + internal_dns 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb expunged fd00:1122:3344:2::1 + internal_ntp 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb expunged fd00:1122:3344:103::21 + nexus 67622d61-2df4-414d-aa0e-d1277265f405 expunged fd00:1122:3344:103::22 @@ -146,18 +150,19 @@ WARNING: Zones exist without physical disks! ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::2c - crucible 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:102::23 - crucible 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:102::24 - crucible 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::29 - crucible 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:102::25 - crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::2b - crucible 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::26 - crucible b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::27 - crucible b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:102::28 - crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::2a - internal_ntp f3f2e4f3-0985-4ef6-8336-ce479382d05d expunged fd00:1122:3344:102::21 - nexus 01d58626-e1b0-480f-96be-ac784863c7dc expunged fd00:1122:3344:102::22 + crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::29 + crucible 57b96d5c-b71e-43e4-8869-7d514003d00d expunged fd00:1122:3344:102::2a + crucible 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::26 + crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::28 + crucible 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::23 + crucible b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::24 + crucible b4947d31-f70e-4ee0-8817-0ca6cea9b16b expunged fd00:1122:3344:102::2b + crucible b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:102::25 + crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::27 + crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a expunged fd00:1122:3344:102::2c + internal_dns 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:3::1 + internal_ntp 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:102::21 + nexus 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:102::22 @@ -168,7 +173,7 @@ WARNING: Zones exist without physical disks! METADATA: created by::::::::::: test_blueprint2 created at::::::::::: 1970-01-01T00:00:00.000Z - comment:::::::::::::: sled 48d95fef-bc9f-4f50-9a53-1e075836291d: expunged 12 zones because: sled policy is expunged + comment:::::::::::::: sled 48d95fef-bc9f-4f50-9a53-1e075836291d: expunged 13 zones because: sled policy is expunged internal DNS version: 1 external DNS version: 1 diff --git a/nexus/reconfigurator/preparation/src/lib.rs b/nexus/reconfigurator/preparation/src/lib.rs index fc0e4638f8..7fa22b8441 100644 --- a/nexus/reconfigurator/preparation/src/lib.rs +++ b/nexus/reconfigurator/preparation/src/lib.rs @@ -35,10 +35,12 @@ use omicron_common::address::IpRange; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; use omicron_common::api::external::Error; +use omicron_common::api::external::InternalContext; use omicron_common::api::external::LookupType; use omicron_common::disk::DiskIdentity; use omicron_common::policy::BOUNDARY_NTP_REDUNDANCY; use omicron_common::policy::COCKROACHDB_REDUNDANCY; +use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_common::policy::NEXUS_REDUNDANCY; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; @@ -63,6 +65,7 @@ pub struct PlanningInputFromDb<'a> { pub service_nic_rows: &'a [nexus_db_model::ServiceNetworkInterface], pub target_boundary_ntp_zone_count: usize, pub target_nexus_zone_count: usize, + pub target_internal_dns_zone_count: usize, pub target_cockroachdb_zone_count: usize, pub target_cockroachdb_cluster_version: CockroachDbClusterVersion, pub internal_dns_version: nexus_db_model::Generation, @@ -72,6 +75,75 @@ pub struct PlanningInputFromDb<'a> { } impl PlanningInputFromDb<'_> { + pub async fn assemble( + opctx: &OpContext, + datastore: &DataStore, + ) -> Result { + opctx.check_complex_operations_allowed()?; + let sled_rows = datastore + .sled_list_all_batched(opctx, SledFilter::Commissioned) + .await + .internal_context("fetching all sleds")?; + let zpool_rows = datastore + .zpool_list_all_external_batched(opctx) + .await + .internal_context("fetching all external zpool rows")?; + let ip_pool_range_rows = { + let (authz_service_ip_pool, _) = datastore + .ip_pools_service_lookup(opctx) + .await + .internal_context("fetching IP services pool")?; + datastore + .ip_pool_list_ranges_batched(opctx, &authz_service_ip_pool) + .await + .internal_context("listing services IP pool ranges")? + }; + let external_ip_rows = datastore + .external_ip_list_service_all_batched(opctx) + .await + .internal_context("fetching service external IPs")?; + let service_nic_rows = datastore + .service_network_interfaces_all_list_batched(opctx) + .await + .internal_context("fetching service NICs")?; + let internal_dns_version = datastore + .dns_group_latest_version(opctx, DnsGroup::Internal) + .await + .internal_context("fetching internal DNS version")? + .version; + let external_dns_version = datastore + .dns_group_latest_version(opctx, DnsGroup::External) + .await + .internal_context("fetching external DNS version")? + .version; + let cockroachdb_settings = datastore + .cockroachdb_settings(opctx) + .await + .internal_context("fetching cockroachdb settings")?; + + let planning_input = PlanningInputFromDb { + sled_rows: &sled_rows, + zpool_rows: &zpool_rows, + ip_pool_range_rows: &ip_pool_range_rows, + target_boundary_ntp_zone_count: BOUNDARY_NTP_REDUNDANCY, + target_nexus_zone_count: NEXUS_REDUNDANCY, + target_internal_dns_zone_count: INTERNAL_DNS_REDUNDANCY, + target_cockroachdb_zone_count: COCKROACHDB_REDUNDANCY, + target_cockroachdb_cluster_version: + CockroachDbClusterVersion::POLICY, + external_ip_rows: &external_ip_rows, + service_nic_rows: &service_nic_rows, + log: &opctx.log, + internal_dns_version, + external_dns_version, + cockroachdb_settings: &cockroachdb_settings, + } + .build() + .internal_context("assembling planning_input")?; + + Ok(planning_input) + } + pub fn build(&self) -> Result { let service_ip_pool_ranges = self.ip_pool_range_rows.iter().map(IpRange::from).collect(); @@ -79,6 +151,7 @@ impl PlanningInputFromDb<'_> { service_ip_pool_ranges, target_boundary_ntp_zone_count: self.target_boundary_ntp_zone_count, target_nexus_zone_count: self.target_nexus_zone_count, + target_internal_dns_zone_count: self.target_internal_dns_zone_count, target_cockroachdb_zone_count: self.target_cockroachdb_zone_count, target_cockroachdb_cluster_version: self .target_cockroachdb_cluster_version, @@ -195,65 +268,8 @@ pub async fn reconfigurator_state_load( datastore: &DataStore, ) -> Result { opctx.check_complex_operations_allowed()?; - let sled_rows = datastore - .sled_list_all_batched(opctx, SledFilter::Commissioned) - .await - .context("listing sleds")?; - let zpool_rows = datastore - .zpool_list_all_external_batched(opctx) - .await - .context("listing zpools")?; - let ip_pool_range_rows = { - let (authz_service_ip_pool, _) = datastore - .ip_pools_service_lookup(opctx) - .await - .context("fetching IP services pool")?; - datastore - .ip_pool_list_ranges_batched(opctx, &authz_service_ip_pool) - .await - .context("listing services IP pool ranges")? - }; - let external_ip_rows = datastore - .external_ip_list_service_all_batched(opctx) - .await - .context("fetching service external IPs")?; - let service_nic_rows = datastore - .service_network_interfaces_all_list_batched(opctx) - .await - .context("fetching service NICs")?; - let internal_dns_version = datastore - .dns_group_latest_version(opctx, DnsGroup::Internal) - .await - .context("fetching internal DNS version")? - .version; - let external_dns_version = datastore - .dns_group_latest_version(opctx, DnsGroup::External) - .await - .context("fetching external DNS version")? - .version; - let cockroachdb_settings = datastore - .cockroachdb_settings(opctx) - .await - .context("fetching cockroachdb settings")?; - - let planning_input = PlanningInputFromDb { - sled_rows: &sled_rows, - zpool_rows: &zpool_rows, - ip_pool_range_rows: &ip_pool_range_rows, - target_boundary_ntp_zone_count: BOUNDARY_NTP_REDUNDANCY, - target_nexus_zone_count: NEXUS_REDUNDANCY, - target_cockroachdb_zone_count: COCKROACHDB_REDUNDANCY, - target_cockroachdb_cluster_version: CockroachDbClusterVersion::POLICY, - external_ip_rows: &external_ip_rows, - service_nic_rows: &service_nic_rows, - log: &opctx.log, - internal_dns_version, - external_dns_version, - cockroachdb_settings: &cockroachdb_settings, - } - .build() - .context("assembling planning_input")?; - + let planning_input = + PlanningInputFromDb::assemble(opctx, datastore).await?; let collection_ids = datastore .inventory_collections() .await diff --git a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs index a81080ec75..ca6e7e4271 100644 --- a/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs +++ b/nexus/src/app/background/tasks/abandoned_vmm_reaper.rs @@ -28,8 +28,8 @@ //! remains alive and continues to own its virtual provisioning resources. //! //! Cleanup of instance resources when an instance's *active* VMM is destroyed -//! is handled elsewhere, by `notify_instance_updated` and (eventually) the -//! `instance-update` saga. +//! is handled elsewhere, by `process_vmm_update` and the `instance-update` +//! saga. use crate::app::background::BackgroundTask; use anyhow::Context; diff --git a/nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs b/nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs index 602f3f85e8..6e49ddc7f0 100644 --- a/nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs +++ b/nexus/src/app/background/tasks/decommissioned_disk_cleaner.rs @@ -179,13 +179,13 @@ mod tests { use diesel::ExpressionMethods; use diesel::QueryDsl; use nexus_db_model::Dataset; - use nexus_db_model::DatasetKind; use nexus_db_model::PhysicalDisk; use nexus_db_model::PhysicalDiskKind; use nexus_db_model::PhysicalDiskPolicy; use nexus_db_model::Region; use nexus_test_utils::SLED_AGENT_UUID; use nexus_test_utils_macros::nexus_test; + use omicron_common::api::internal::shared::DatasetKind; use omicron_uuid_kinds::{ DatasetUuid, PhysicalDiskUuid, RegionUuid, SledUuid, }; diff --git a/nexus/src/app/background/tasks/instance_watcher.rs b/nexus/src/app/background/tasks/instance_watcher.rs index f63c21105e..ae78392ea3 100644 --- a/nexus/src/app/background/tasks/instance_watcher.rs +++ b/nexus/src/app/background/tasks/instance_watcher.rs @@ -19,9 +19,9 @@ use nexus_types::identity::Asset; use nexus_types::identity::Resource; use omicron_common::api::external::Error; use omicron_common::api::external::InstanceState; -use omicron_common::api::internal::nexus::SledInstanceState; +use omicron_common::api::internal::nexus::SledVmmState; use omicron_uuid_kinds::GenericUuid; -use omicron_uuid_kinds::InstanceUuid; +use omicron_uuid_kinds::PropolisUuid; use oximeter::types::ProducerRegistry; use sled_agent_client::Client as SledAgentClient; use std::borrow::Cow; @@ -81,12 +81,12 @@ impl InstanceWatcher { let client = client.clone(); async move { - slog::trace!(opctx.log, "checking on instance..."); - let rsp = client - .instance_get_state(&InstanceUuid::from_untyped_uuid( - target.instance_id, - )) - .await; + let vmm_id = PropolisUuid::from_untyped_uuid(target.vmm_id); + slog::trace!( + opctx.log, "checking on VMM"; "propolis_id" => %vmm_id + ); + + let rsp = client.vmm_get_state(&vmm_id).await; let mut check = Check { target, outcome: Default::default(), @@ -151,7 +151,7 @@ impl InstanceWatcher { } }; - let new_runtime_state: SledInstanceState = state.into(); + let new_runtime_state: SledVmmState = state.into(); check.outcome = CheckOutcome::Success(new_runtime_state.vmm_state.state.into()); debug!( @@ -159,10 +159,10 @@ impl InstanceWatcher { "updating instance state"; "state" => ?new_runtime_state.vmm_state.state, ); - match crate::app::instance::notify_instance_updated( + match crate::app::instance::process_vmm_update( &datastore, &opctx, - InstanceUuid::from_untyped_uuid(target.instance_id), + PropolisUuid::from_untyped_uuid(target.vmm_id), &new_runtime_state, ) .await @@ -176,7 +176,7 @@ impl InstanceWatcher { _ => Err(Incomplete::UpdateFailed), }; } - Ok(Some(saga)) => { + Ok(Some((_, saga))) => { check.update_saga_queued = true; if let Err(e) = sagas.saga_start(saga).await { warn!(opctx.log, "update saga failed"; "error" => ?e); diff --git a/nexus/src/app/background/tasks/saga_recovery.rs b/nexus/src/app/background/tasks/saga_recovery.rs index 7b0fe1b331..42069ac4ed 100644 --- a/nexus/src/app/background/tasks/saga_recovery.rs +++ b/nexus/src/app/background/tasks/saga_recovery.rs @@ -517,7 +517,7 @@ mod test { ) -> (dev::db::CockroachInstance, Arc) { let db = test_setup_database(&log).await; let cfg = nexus_db_queries::db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(db::Pool::new(log, &cfg)); + let pool = Arc::new(db::Pool::new_single_host(log, &cfg)); let db_datastore = Arc::new( db::DataStore::new(&log, Arc::clone(&pool), None).await.unwrap(), ); diff --git a/nexus/src/app/deployment.rs b/nexus/src/app/deployment.rs index 50ae332d3f..79e7a93e6d 100644 --- a/nexus/src/app/deployment.rs +++ b/nexus/src/app/deployment.rs @@ -4,7 +4,6 @@ //! Configuration of the deployment system -use nexus_db_model::DnsGroup; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_reconfigurator_planning::planner::Planner; @@ -13,9 +12,7 @@ use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintMetadata; use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::BlueprintTargetSet; -use nexus_types::deployment::CockroachDbClusterVersion; use nexus_types::deployment::PlanningInput; -use nexus_types::deployment::SledFilter; use nexus_types::inventory::Collection; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; @@ -25,9 +22,6 @@ use omicron_common::api::external::InternalContext; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::LookupType; -use omicron_common::policy::BOUNDARY_NTP_REDUNDANCY; -use omicron_common::policy::COCKROACHDB_REDUNDANCY; -use omicron_common::policy::NEXUS_REDUNDANCY; use slog_error_chain::InlineErrorChain; use uuid::Uuid; @@ -132,61 +126,8 @@ impl super::Nexus { ) -> Result { let creator = self.id.to_string(); let datastore = self.datastore(); - - let sled_rows = datastore - .sled_list_all_batched(opctx, SledFilter::Commissioned) - .await?; - let zpool_rows = - datastore.zpool_list_all_external_batched(opctx).await?; - let ip_pool_range_rows = { - let (authz_service_ip_pool, _) = - datastore.ip_pools_service_lookup(opctx).await?; - datastore - .ip_pool_list_ranges_batched(opctx, &authz_service_ip_pool) - .await? - }; - let external_ip_rows = - datastore.external_ip_list_service_all_batched(opctx).await?; - let service_nic_rows = datastore - .service_network_interfaces_all_list_batched(opctx) - .await?; - - let internal_dns_version = datastore - .dns_group_latest_version(opctx, DnsGroup::Internal) - .await - .internal_context( - "fetching internal DNS version for blueprint planning", - )? - .version; - let external_dns_version = datastore - .dns_group_latest_version(opctx, DnsGroup::External) - .await - .internal_context( - "fetching external DNS version for blueprint planning", - )? - .version; - let cockroachdb_settings = - datastore.cockroachdb_settings(opctx).await.internal_context( - "fetching cockroachdb settings for blueprint planning", - )?; - - let planning_input = PlanningInputFromDb { - sled_rows: &sled_rows, - zpool_rows: &zpool_rows, - ip_pool_range_rows: &ip_pool_range_rows, - external_ip_rows: &external_ip_rows, - service_nic_rows: &service_nic_rows, - target_boundary_ntp_zone_count: BOUNDARY_NTP_REDUNDANCY, - target_nexus_zone_count: NEXUS_REDUNDANCY, - target_cockroachdb_zone_count: COCKROACHDB_REDUNDANCY, - target_cockroachdb_cluster_version: - CockroachDbClusterVersion::POLICY, - log: &opctx.log, - internal_dns_version, - external_dns_version, - cockroachdb_settings: &cockroachdb_settings, - } - .build()?; + let planning_input = + PlanningInputFromDb::assemble(opctx, datastore).await?; // The choice of which inventory collection to use here is not // necessarily trivial. Inventory collections may be incomplete due to diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 3106ab9f2a..b715b6bbd3 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -60,7 +60,7 @@ use propolis_client::support::WebSocketStream; use sagas::instance_common::ExternalIpAttach; use sled_agent_client::types::InstanceMigrationTargetParams; use sled_agent_client::types::InstanceProperties; -use sled_agent_client::types::InstancePutStateBody; +use sled_agent_client::types::VmmPutStateBody; use std::matches; use std::net::SocketAddr; use std::sync::Arc; @@ -154,7 +154,7 @@ pub(crate) enum InstanceStateChangeRequest { } impl From - for sled_agent_client::types::InstanceStateRequested + for sled_agent_client::types::VmmStateRequested { fn from(value: InstanceStateChangeRequest) -> Self { match value { @@ -176,7 +176,7 @@ enum InstanceStateChangeRequestAction { /// Request the appropriate state change from the sled with the specified /// UUID. - SendToSled(SledUuid), + SendToSled { sled_id: SledUuid, propolis_id: PropolisUuid }, } /// What is the higher level operation that is calling @@ -553,7 +553,6 @@ impl super::Nexus { if let Err(e) = self .instance_request_state( opctx, - &authz_instance, state.instance(), state.vmm(), InstanceStateChangeRequest::Reboot, @@ -632,7 +631,6 @@ impl super::Nexus { if let Err(e) = self .instance_request_state( opctx, - &authz_instance, state.instance(), state.vmm(), InstanceStateChangeRequest::Stop, @@ -664,21 +662,18 @@ impl super::Nexus { /// this sled, this operation rudely terminates it. pub(crate) async fn instance_ensure_unregistered( &self, - opctx: &OpContext, - authz_instance: &authz::Instance, + propolis_id: &PropolisUuid, sled_id: &SledUuid, - ) -> Result, InstanceStateChangeError> - { - opctx.authorize(authz::Action::Modify, authz_instance).await?; + ) -> Result, InstanceStateChangeError> { let sa = self.sled_client(&sled_id).await?; - sa.instance_unregister(&InstanceUuid::from_untyped_uuid( - authz_instance.id(), - )) - .await - .map(|res| res.into_inner().updated_runtime.map(Into::into)) - .map_err(|e| { - InstanceStateChangeError::SledAgent(SledAgentInstancePutError(e)) - }) + sa.vmm_unregister(propolis_id) + .await + .map(|res| res.into_inner().updated_runtime.map(Into::into)) + .map_err(|e| { + InstanceStateChangeError::SledAgent(SledAgentInstancePutError( + e, + )) + }) } /// Determines the action to take on an instance's active VMM given a @@ -712,8 +707,11 @@ impl super::Nexus { // Requests that operate on active instances have to be directed to the // instance's current sled agent. If there is none, the request needs to // be handled specially based on its type. - let sled_id = if let Some(vmm) = vmm_state { - SledUuid::from_untyped_uuid(vmm.sled_id) + let (sled_id, propolis_id) = if let Some(vmm) = vmm_state { + ( + SledUuid::from_untyped_uuid(vmm.sled_id), + PropolisUuid::from_untyped_uuid(vmm.id), + ) } else { match effective_state { // If there's no active sled because the instance is stopped, @@ -814,7 +812,10 @@ impl super::Nexus { }; if allowed { - Ok(InstanceStateChangeRequestAction::SendToSled(sled_id)) + Ok(InstanceStateChangeRequestAction::SendToSled { + sled_id, + propolis_id, + }) } else { Err(Error::invalid_request(format!( "instance state cannot be changed from state \"{}\"", @@ -826,26 +827,25 @@ impl super::Nexus { pub(crate) async fn instance_request_state( &self, opctx: &OpContext, - authz_instance: &authz::Instance, prev_instance_state: &db::model::Instance, prev_vmm_state: &Option, requested: InstanceStateChangeRequest, ) -> Result<(), InstanceStateChangeError> { - opctx.authorize(authz::Action::Modify, authz_instance).await?; - let instance_id = InstanceUuid::from_untyped_uuid(authz_instance.id()); - match self.select_runtime_change_action( prev_instance_state, prev_vmm_state, &requested, )? { InstanceStateChangeRequestAction::AlreadyDone => Ok(()), - InstanceStateChangeRequestAction::SendToSled(sled_id) => { + InstanceStateChangeRequestAction::SendToSled { + sled_id, + propolis_id, + } => { let sa = self.sled_client(&sled_id).await?; let instance_put_result = sa - .instance_put_state( - &instance_id, - &InstancePutStateBody { state: requested.into() }, + .vmm_put_state( + &propolis_id, + &VmmPutStateBody { state: requested.into() }, ) .await .map(|res| res.into_inner().updated_runtime.map(Into::into)) @@ -862,7 +862,7 @@ impl super::Nexus { // Ok(None) here, in which case, there's nothing to write back. match instance_put_result { Ok(Some(ref state)) => self - .notify_instance_updated(opctx, instance_id, state) + .notify_vmm_updated(opctx, propolis_id, state) .await .map_err(Into::into), Ok(None) => Ok(()), @@ -1120,13 +1120,13 @@ impl super::Nexus { .sled_client(&SledUuid::from_untyped_uuid(initial_vmm.sled_id)) .await?; let instance_register_result = sa - .instance_register( - &instance_id, + .vmm_register( + propolis_id, &sled_agent_client::types::InstanceEnsureBody { hardware: instance_hardware, instance_runtime: db_instance.runtime().clone().into(), vmm_runtime: initial_vmm.clone().into(), - propolis_id: *propolis_id, + instance_id, propolis_addr: SocketAddr::new( initial_vmm.propolis_ip.ip(), initial_vmm.propolis_port.into(), @@ -1141,8 +1141,7 @@ impl super::Nexus { match instance_register_result { Ok(state) => { - self.notify_instance_updated(opctx, instance_id, &state) - .await?; + self.notify_vmm_updated(opctx, *propolis_id, &state).await?; } Err(e) => { if e.instance_unhealthy() { @@ -1321,19 +1320,22 @@ impl super::Nexus { /// Invoked by a sled agent to publish an updated runtime state for an /// Instance. - pub(crate) async fn notify_instance_updated( + pub(crate) async fn notify_vmm_updated( &self, opctx: &OpContext, - instance_id: InstanceUuid, - new_runtime_state: &nexus::SledInstanceState, + propolis_id: PropolisUuid, + new_runtime_state: &nexus::SledVmmState, ) -> Result<(), Error> { - let saga = notify_instance_updated( + let Some((instance_id, saga)) = process_vmm_update( &self.db_datastore, opctx, - instance_id, + propolis_id, new_runtime_state, ) - .await?; + .await? + else { + return Ok(()); + }; // We don't need to wait for the instance update saga to run to // completion to return OK to the sled-agent --- all it needs to care @@ -1344,53 +1346,51 @@ impl super::Nexus { // one is eventually executed. // // Therefore, just spawn the update saga in a new task, and return. - if let Some(saga) = saga { - info!(opctx.log, "starting update saga for {instance_id}"; - "instance_id" => %instance_id, - "vmm_state" => ?new_runtime_state.vmm_state, - "migration_state" => ?new_runtime_state.migrations(), - ); - let sagas = self.sagas.clone(); - let task_instance_updater = - self.background_tasks.task_instance_updater.clone(); - let log = opctx.log.clone(); - tokio::spawn(async move { - // TODO(eliza): maybe we should use the lower level saga API so - // we can see if the saga failed due to the lock being held and - // retry it immediately? - let running_saga = async move { - let runnable_saga = sagas.saga_prepare(saga).await?; - runnable_saga.start().await - } - .await; - let result = match running_saga { - Err(error) => { - error!(&log, "failed to start update saga for {instance_id}"; - "instance_id" => %instance_id, - "error" => %error, - ); - // If we couldn't start the update saga for this - // instance, kick the instance-updater background task - // to try and start it again in a timely manner. - task_instance_updater.activate(); - return; - } - Ok(saga) => { - saga.wait_until_stopped().await.into_omicron_result() - } - }; - if let Err(error) = result { - error!(&log, "update saga for {instance_id} failed"; + info!(opctx.log, "starting update saga for {instance_id}"; + "instance_id" => %instance_id, + "vmm_state" => ?new_runtime_state.vmm_state, + "migration_state" => ?new_runtime_state.migrations(), + ); + let sagas = self.sagas.clone(); + let task_instance_updater = + self.background_tasks.task_instance_updater.clone(); + let log = opctx.log.clone(); + tokio::spawn(async move { + // TODO(eliza): maybe we should use the lower level saga API so + // we can see if the saga failed due to the lock being held and + // retry it immediately? + let running_saga = async move { + let runnable_saga = sagas.saga_prepare(saga).await?; + runnable_saga.start().await + } + .await; + let result = match running_saga { + Err(error) => { + error!(&log, "failed to start update saga for {instance_id}"; "instance_id" => %instance_id, "error" => %error, ); - // If we couldn't complete the update saga for this + // If we couldn't start the update saga for this // instance, kick the instance-updater background task // to try and start it again in a timely manner. task_instance_updater.activate(); + return; } - }); - } + Ok(saga) => { + saga.wait_until_stopped().await.into_omicron_result() + } + }; + if let Err(error) = result { + error!(&log, "update saga for {instance_id} failed"; + "instance_id" => %instance_id, + "error" => %error, + ); + // If we couldn't complete the update saga for this + // instance, kick the instance-updater background task + // to try and start it again in a timely manner. + task_instance_updater.activate(); + } + }); Ok(()) } @@ -1830,21 +1830,27 @@ impl super::Nexus { } } -/// Invoked by a sled agent to publish an updated runtime state for an -/// Instance, returning an update saga for that instance (if one must be -/// executed). -pub(crate) async fn notify_instance_updated( +/// Writes the VMM and migration state supplied in `new_runtime_state` to the +/// database (provided that it's newer than what's already there). +/// +/// # Return value +/// +/// - `Ok(Some(instance_id, saga))` if the new VMM state obsoletes the current +/// instance state. The caller should execute the returned instance update +/// saga to reconcile the instance to the new VMM state. +/// - `Ok(None)` if the new state was successfully published but does not +/// require an instance update. +/// - `Err` if an error occurred. +pub(crate) async fn process_vmm_update( datastore: &DataStore, opctx: &OpContext, - instance_id: InstanceUuid, - new_runtime_state: &nexus::SledInstanceState, -) -> Result, Error> { + propolis_id: PropolisUuid, + new_runtime_state: &nexus::SledVmmState, +) -> Result, Error> { use sagas::instance_update; let migrations = new_runtime_state.migrations(); - let propolis_id = new_runtime_state.propolis_id; info!(opctx.log, "received new VMM runtime state from sled agent"; - "instance_id" => %instance_id, "propolis_id" => %propolis_id, "vmm_state" => ?new_runtime_state.vmm_state, "migration_state" => ?migrations, @@ -1864,21 +1870,34 @@ pub(crate) async fn notify_instance_updated( // prepare and return it. if instance_update::update_saga_needed( &opctx.log, - instance_id, + propolis_id, new_runtime_state, &result, ) { + let instance_id = + InstanceUuid::from_untyped_uuid(result.found_vmm.instance_id); + let (.., authz_instance) = LookupPath::new(&opctx, datastore) .instance_id(instance_id.into_untyped_uuid()) .lookup_for(authz::Action::Modify) .await?; - let saga = instance_update::SagaInstanceUpdate::prepare( + + match instance_update::SagaInstanceUpdate::prepare( &instance_update::Params { serialized_authn: authn::saga::Serialized::for_opctx(opctx), authz_instance, }, - )?; - Ok(Some(saga)) + ) { + Ok(saga) => Ok(Some((instance_id, saga))), + Err(e) => { + error!(opctx.log, "failed to prepare instance update saga"; + "error" => ?e, + "instance_id" => %instance_id, + "propolis_id" => %propolis_id); + + Err(e) + } + } } else { Ok(None) } diff --git a/nexus/src/app/metrics.rs b/nexus/src/app/metrics.rs index 3a6e7e27be..4dc7309e76 100644 --- a/nexus/src/app/metrics.rs +++ b/nexus/src/app/metrics.rs @@ -4,7 +4,6 @@ //! Metrics -use crate::external_api::http_entrypoints::SystemMetricName; use crate::external_api::params::ResourceMetrics; use dropshot::PaginationParams; use nexus_db_queries::authz; @@ -12,10 +11,10 @@ use nexus_db_queries::{ context::OpContext, db::{fixed_data::FLEET_ID, lookup}, }; +use nexus_external_api::TimeseriesSchemaPaginationParams; +use nexus_types::external_api::params::SystemMetricName; use omicron_common::api::external::{Error, InternalContext}; -use oximeter_db::{ - Measurement, TimeseriesSchema, TimeseriesSchemaPaginationParams, -}; +use oximeter_db::{Measurement, TimeseriesSchema}; use std::num::NonZeroU32; impl super::Nexus { diff --git a/nexus/src/app/oximeter.rs b/nexus/src/app/oximeter.rs index 9039d1b8fa..0c7ec3a016 100644 --- a/nexus/src/app/oximeter.rs +++ b/nexus/src/app/oximeter.rs @@ -12,7 +12,7 @@ use internal_dns::ServiceName; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use nexus_db_queries::db::DataStore; -use omicron_common::address::CLICKHOUSE_PORT; +use omicron_common::address::CLICKHOUSE_HTTP_PORT; use omicron_common::api::external::Error; use omicron_common::api::external::{DataPageParams, ListResultVec}; use omicron_common::api::internal::nexus::{self, ProducerEndpoint}; @@ -65,7 +65,7 @@ impl LazyTimeseriesClient { ClientSource::FromIp { address } => *address, ClientSource::FromDns { resolver } => SocketAddr::new( resolver.lookup_ip(ServiceName::Clickhouse).await?, - CLICKHOUSE_PORT, + CLICKHOUSE_HTTP_PORT, ), }; diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index f3c0031327..835541c2ea 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -147,7 +147,7 @@ impl super::Nexus { dataset.dataset_id, dataset.zpool_id, Some(dataset.request.address), - dataset.request.kind.into(), + dataset.request.kind, ) }) .collect(); diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs index 6e431aaca7..049673d2ee 100644 --- a/nexus/src/app/sagas/instance_common.rs +++ b/nexus/src/app/sagas/instance_common.rs @@ -25,6 +25,12 @@ use super::NexusActionContext; /// The port propolis-server listens on inside the propolis zone. const DEFAULT_PROPOLIS_PORT: u16 = 12400; +#[derive(Clone, Debug, Serialize, Deserialize)] +pub(super) struct VmmAndSledIds { + pub(super) vmm_id: PropolisUuid, + pub(super) sled_id: SledUuid, +} + /// Reserves resources for a new VMM whose instance has `ncpus` guest logical /// processors and `guest_memory` bytes of guest RAM. The selected sled is /// random within the set of sleds allowed by the supplied `constraints`. @@ -213,12 +219,12 @@ pub async fn instance_ip_move_state( /// the Attaching or Detaching state so that concurrent attempts to start the /// instance will notice that the IP state is in flux and ask the caller to /// retry. -pub async fn instance_ip_get_instance_state( +pub(super) async fn instance_ip_get_instance_state( sagactx: &NexusActionContext, serialized_authn: &authn::saga::Serialized, authz_instance: &authz::Instance, verb: &str, -) -> Result, ActionError> { +) -> Result, ActionError> { // XXX: we can get instance state (but not sled ID) in same transaction // as attach (but not detach) wth current design. We need to re-query // for sled ID anyhow, so keep consistent between attach/detach. @@ -236,7 +242,11 @@ pub async fn instance_ip_get_instance_state( inst_and_vmm.vmm().as_ref().map(|vmm| vmm.runtime.state); let found_instance_state = inst_and_vmm.instance().runtime_state.nexus_state; - let mut sled_id = inst_and_vmm.sled_id(); + let mut propolis_and_sled_id = + inst_and_vmm.vmm().as_ref().map(|vmm| VmmAndSledIds { + vmm_id: PropolisUuid::from_untyped_uuid(vmm.id), + sled_id: SledUuid::from_untyped_uuid(vmm.sled_id), + }); slog::debug!( osagactx.log(), "evaluating instance state for IP attach/detach"; @@ -257,7 +267,7 @@ pub async fn instance_ip_get_instance_state( match (found_instance_state, found_vmm_state) { // If there's no VMM, the instance is definitely not on any sled. (InstanceState::NoVmm, _) | (_, Some(VmmState::SagaUnwound)) => { - sled_id = None; + propolis_and_sled_id = None; } // If the instance is running normally or rebooting, it's resident on @@ -340,7 +350,7 @@ pub async fn instance_ip_get_instance_state( } } - Ok(sled_id) + Ok(propolis_and_sled_id) } /// Adds a NAT entry to DPD, routing packets bound for `target_ip` to a @@ -441,18 +451,19 @@ pub async fn instance_ip_remove_nat( /// Inform the OPTE port for a running instance that it should start /// sending/receiving traffic on a given IP address. /// -/// This call is a no-op if `sled_uuid` is `None` or the saga is explicitly -/// set to be inactive in event of double attach/detach (`!target_ip.do_saga`). -pub async fn instance_ip_add_opte( +/// This call is a no-op if the instance is not active (`propolis_and_sled` is +/// `None`) or the calling saga is explicitly set to be inactive in the event of +/// a double attach/detach (`!target_ip.do_saga`). +pub(super) async fn instance_ip_add_opte( sagactx: &NexusActionContext, - authz_instance: &authz::Instance, - sled_uuid: Option, + vmm_and_sled: Option, target_ip: ModifyStateForExternalIp, ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); // No physical sled? Don't inform OPTE. - let Some(sled_uuid) = sled_uuid else { + let Some(VmmAndSledIds { vmm_id: propolis_id, sled_id }) = vmm_and_sled + else { return Ok(()); }; @@ -470,17 +481,14 @@ pub async fn instance_ip_add_opte( osagactx .nexus() - .sled_client(&sled_uuid) + .sled_client(&sled_id) .await .map_err(|_| { ActionError::action_failed(Error::unavail( "sled agent client went away mid-attach/detach", )) })? - .instance_put_external_ip( - &InstanceUuid::from_untyped_uuid(authz_instance.id()), - &sled_agent_body, - ) + .vmm_put_external_ip(&propolis_id, &sled_agent_body) .await .map_err(|e| { ActionError::action_failed(match e { @@ -499,18 +507,20 @@ pub async fn instance_ip_add_opte( /// Inform the OPTE port for a running instance that it should cease /// sending/receiving traffic on a given IP address. /// -/// This call is a no-op if `sled_uuid` is `None` or the saga is explicitly -/// set to be inactive in event of double attach/detach (`!target_ip.do_saga`). -pub async fn instance_ip_remove_opte( +/// This call is a no-op if the instance is not active (`propolis_and_sled` is +/// `None`) or the calling saga is explicitly set to be inactive in the event of +/// a double attach/detach (`!target_ip.do_saga`). +pub(super) async fn instance_ip_remove_opte( sagactx: &NexusActionContext, - authz_instance: &authz::Instance, - sled_uuid: Option, + propolis_and_sled: Option, target_ip: ModifyStateForExternalIp, ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); // No physical sled? Don't inform OPTE. - let Some(sled_uuid) = sled_uuid else { + let Some(VmmAndSledIds { vmm_id: propolis_id, sled_id }) = + propolis_and_sled + else { return Ok(()); }; @@ -528,17 +538,14 @@ pub async fn instance_ip_remove_opte( osagactx .nexus() - .sled_client(&sled_uuid) + .sled_client(&sled_id) .await .map_err(|_| { ActionError::action_failed(Error::unavail( "sled agent client went away mid-attach/detach", )) })? - .instance_delete_external_ip( - &InstanceUuid::from_untyped_uuid(authz_instance.id()), - &sled_agent_body, - ) + .vmm_delete_external_ip(&propolis_id, &sled_agent_body) .await .map_err(|e| { ActionError::action_failed(match e { diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index d19230892f..0b6d8cc0f8 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -1220,8 +1220,7 @@ pub mod test { } async fn no_instances_or_disks_on_sled(sled_agent: &SledAgent) -> bool { - sled_agent.instance_count().await == 0 - && sled_agent.disk_count().await == 0 + sled_agent.vmm_count().await == 0 && sled_agent.disk_count().await == 0 } pub(crate) async fn verify_clean_slate( diff --git a/nexus/src/app/sagas/instance_ip_attach.rs b/nexus/src/app/sagas/instance_ip_attach.rs index a14054cf66..e6fb8654ea 100644 --- a/nexus/src/app/sagas/instance_ip_attach.rs +++ b/nexus/src/app/sagas/instance_ip_attach.rs @@ -5,7 +5,7 @@ use super::instance_common::{ instance_ip_add_nat, instance_ip_add_opte, instance_ip_get_instance_state, instance_ip_move_state, instance_ip_remove_opte, ExternalIpAttach, - ModifyStateForExternalIp, + ModifyStateForExternalIp, VmmAndSledIds, }; use super::{ActionRegistry, NexusActionContext, NexusSaga}; use crate::app::sagas::declare_saga_actions; @@ -13,7 +13,7 @@ use crate::app::{authn, authz}; use nexus_db_model::{IpAttachState, Ipv4NatEntry}; use nexus_types::external_api::views; use omicron_common::api::external::Error; -use omicron_uuid_kinds::{GenericUuid, InstanceUuid, SledUuid}; +use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; use serde::Deserialize; use serde::Serialize; use steno::ActionError; @@ -161,7 +161,7 @@ async fn siia_begin_attach_ip_undo( async fn siia_get_instance_state( sagactx: NexusActionContext, -) -> Result, ActionError> { +) -> Result, ActionError> { let params = sagactx.saga_params::()?; instance_ip_get_instance_state( &sagactx, @@ -177,7 +177,10 @@ async fn siia_nat( sagactx: NexusActionContext, ) -> Result, ActionError> { let params = sagactx.saga_params::()?; - let sled_id = sagactx.lookup::>("instance_state")?; + let sled_id = sagactx + .lookup::>("instance_state")? + .map(|ids| ids.sled_id); + let target_ip = sagactx.lookup::("target_ip")?; instance_ip_add_nat( &sagactx, @@ -245,28 +248,18 @@ async fn siia_nat_undo( async fn siia_update_opte( sagactx: NexusActionContext, ) -> Result<(), ActionError> { - let params = sagactx.saga_params::()?; - let sled_id = sagactx.lookup::>("instance_state")?; + let ids = sagactx.lookup::>("instance_state")?; let target_ip = sagactx.lookup::("target_ip")?; - instance_ip_add_opte(&sagactx, ¶ms.authz_instance, sled_id, target_ip) - .await + instance_ip_add_opte(&sagactx, ids, target_ip).await } async fn siia_update_opte_undo( sagactx: NexusActionContext, ) -> Result<(), anyhow::Error> { let log = sagactx.user_data().log(); - let params = sagactx.saga_params::()?; - let sled_id = sagactx.lookup::>("instance_state")?; + let ids = sagactx.lookup::>("instance_state")?; let target_ip = sagactx.lookup::("target_ip")?; - if let Err(e) = instance_ip_remove_opte( - &sagactx, - ¶ms.authz_instance, - sled_id, - target_ip, - ) - .await - { + if let Err(e) = instance_ip_remove_opte(&sagactx, ids, target_ip).await { error!(log, "siia_update_opte_undo: failed to notify sled-agent: {e}"); } Ok(()) @@ -436,8 +429,14 @@ pub(crate) mod test { } // Sled agent has a record of the new external IPs. + let VmmAndSledIds { vmm_id, .. } = + crate::app::sagas::test_helpers::instance_fetch_vmm_and_sled_ids( + cptestctx, + &instance_id, + ) + .await; let mut eips = sled_agent.external_ips.lock().await; - let my_eips = eips.entry(instance_id.into_untyped_uuid()).or_default(); + let my_eips = eips.entry(vmm_id).or_default(); assert!(my_eips .iter() .any(|v| matches!(v, InstanceExternalIpBody::Floating(_)))); @@ -458,7 +457,7 @@ pub(crate) mod test { pub(crate) async fn verify_clean_slate( cptestctx: &ControlPlaneTestContext, - instance_id: Uuid, + instance_id: InstanceUuid, ) { use nexus_db_queries::db::schema::external_ip::dsl; @@ -471,7 +470,7 @@ pub(crate) mod test { assert!(dsl::external_ip .filter(dsl::kind.eq(IpKind::Floating)) .filter(dsl::time_deleted.is_null()) - .filter(dsl::parent_id.eq(instance_id)) + .filter(dsl::parent_id.eq(instance_id.into_untyped_uuid())) .filter(dsl::state.ne(IpAttachState::Detached)) .select(ExternalIp::as_select()) .first_async::(&*conn) @@ -492,8 +491,14 @@ pub(crate) mod test { .is_none()); // No IP bindings remain on sled-agent. + let VmmAndSledIds { vmm_id, .. } = + crate::app::sagas::test_helpers::instance_fetch_vmm_and_sled_ids( + cptestctx, + &instance_id, + ) + .await; let mut eips = sled_agent.external_ips.lock().await; - let my_eips = eips.entry(instance_id).or_default(); + let my_eips = eips.entry(vmm_id).or_default(); assert!(my_eips.is_empty()); } @@ -512,9 +517,10 @@ pub(crate) mod test { let instance = create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); crate::app::sagas::test_helpers::instance_simulate( cptestctx, - &InstanceUuid::from_untyped_uuid(instance.identity.id), + &instance_id, ) .await; @@ -522,7 +528,7 @@ pub(crate) mod test { test_helpers::action_failure_can_unwind::( nexus, || Box::pin(new_test_params(&opctx, datastore, use_float) ), - || Box::pin(verify_clean_slate(&cptestctx, instance.id())), + || Box::pin(verify_clean_slate(&cptestctx, instance_id)), log, ) .await; @@ -544,9 +550,10 @@ pub(crate) mod test { let instance = create_instance(client, PROJECT_NAME, INSTANCE_NAME).await; + let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); crate::app::sagas::test_helpers::instance_simulate( cptestctx, - &InstanceUuid::from_untyped_uuid(instance.identity.id), + &instance_id, ) .await; @@ -558,7 +565,7 @@ pub(crate) mod test { >( nexus, || Box::pin(new_test_params(&opctx, datastore, use_float)), - || Box::pin(verify_clean_slate(&cptestctx, instance.id())), + || Box::pin(verify_clean_slate(&cptestctx, instance_id)), log, ) .await; diff --git a/nexus/src/app/sagas/instance_ip_detach.rs b/nexus/src/app/sagas/instance_ip_detach.rs index a5b51ce375..d9da9fc05c 100644 --- a/nexus/src/app/sagas/instance_ip_detach.rs +++ b/nexus/src/app/sagas/instance_ip_detach.rs @@ -5,7 +5,7 @@ use super::instance_common::{ instance_ip_add_nat, instance_ip_add_opte, instance_ip_get_instance_state, instance_ip_move_state, instance_ip_remove_nat, instance_ip_remove_opte, - ModifyStateForExternalIp, + ModifyStateForExternalIp, VmmAndSledIds, }; use super::{ActionRegistry, NexusActionContext, NexusSaga}; use crate::app::sagas::declare_saga_actions; @@ -15,7 +15,7 @@ use nexus_db_model::IpAttachState; use nexus_db_queries::db::lookup::LookupPath; use nexus_types::external_api::views; use omicron_common::api::external::NameOrId; -use omicron_uuid_kinds::{GenericUuid, InstanceUuid, SledUuid}; +use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; use ref_cast::RefCast; use serde::Deserialize; use serde::Serialize; @@ -155,7 +155,7 @@ async fn siid_begin_detach_ip_undo( async fn siid_get_instance_state( sagactx: NexusActionContext, -) -> Result, ActionError> { +) -> Result, ActionError> { let params = sagactx.saga_params::()?; instance_ip_get_instance_state( &sagactx, @@ -168,7 +168,9 @@ async fn siid_get_instance_state( async fn siid_nat(sagactx: NexusActionContext) -> Result<(), ActionError> { let params = sagactx.saga_params::()?; - let sled_id = sagactx.lookup::>("instance_state")?; + let sled_id = sagactx + .lookup::>("instance_state")? + .map(|ids| ids.sled_id); let target_ip = sagactx.lookup::("target_ip")?; instance_ip_remove_nat( &sagactx, @@ -184,7 +186,9 @@ async fn siid_nat_undo( ) -> Result<(), anyhow::Error> { let log = sagactx.user_data().log(); let params = sagactx.saga_params::()?; - let sled_id = sagactx.lookup::>("instance_state")?; + let sled_id = sagactx + .lookup::>("instance_state")? + .map(|ids| ids.sled_id); let target_ip = sagactx.lookup::("target_ip")?; if let Err(e) = instance_ip_add_nat( &sagactx, @@ -204,33 +208,18 @@ async fn siid_nat_undo( async fn siid_update_opte( sagactx: NexusActionContext, ) -> Result<(), ActionError> { - let params = sagactx.saga_params::()?; - let sled_id = sagactx.lookup::>("instance_state")?; + let ids = sagactx.lookup::>("instance_state")?; let target_ip = sagactx.lookup::("target_ip")?; - instance_ip_remove_opte( - &sagactx, - ¶ms.authz_instance, - sled_id, - target_ip, - ) - .await + instance_ip_remove_opte(&sagactx, ids, target_ip).await } async fn siid_update_opte_undo( sagactx: NexusActionContext, ) -> Result<(), anyhow::Error> { let log = sagactx.user_data().log(); - let params = sagactx.saga_params::()?; - let sled_id = sagactx.lookup::>("instance_state")?; + let ids = sagactx.lookup::>("instance_state")?; let target_ip = sagactx.lookup::("target_ip")?; - if let Err(e) = instance_ip_add_opte( - &sagactx, - ¶ms.authz_instance, - sled_id, - target_ip, - ) - .await - { + if let Err(e) = instance_ip_add_opte(&sagactx, ids, target_ip).await { error!(log, "siid_update_opte_undo: failed to notify sled-agent: {e}"); } Ok(()) @@ -410,8 +399,14 @@ pub(crate) mod test { } // Sled agent has removed its records of the external IPs. + let VmmAndSledIds { vmm_id, .. } = + crate::app::sagas::test_helpers::instance_fetch_vmm_and_sled_ids( + cptestctx, + &instance_id, + ) + .await; let mut eips = sled_agent.external_ips.lock().await; - let my_eips = eips.entry(instance_id.into_untyped_uuid()).or_default(); + let my_eips = eips.entry(vmm_id).or_default(); assert!(my_eips.is_empty()); // DB only has record for SNAT. diff --git a/nexus/src/app/sagas/instance_migrate.rs b/nexus/src/app/sagas/instance_migrate.rs index 19bef2f046..24d11fcae2 100644 --- a/nexus/src/app/sagas/instance_migrate.rs +++ b/nexus/src/app/sagas/instance_migrate.rs @@ -437,20 +437,10 @@ async fn sim_ensure_destination_propolis_undo( sagactx: NexusActionContext, ) -> Result<(), anyhow::Error> { let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - + let dst_propolis_id = sagactx.lookup::("dst_propolis_id")?; let dst_sled_id = sagactx.lookup::("dst_sled_id")?; let db_instance = sagactx.lookup::("set_migration_ids")?; - let (.., authz_instance) = LookupPath::new(&opctx, &osagactx.datastore()) - .instance_id(db_instance.id()) - .lookup_for(authz::Action::Modify) - .await - .map_err(ActionError::action_failed)?; info!(osagactx.log(), "unregistering destination vmm for migration unwind"; "instance_id" => %db_instance.id(), @@ -465,7 +455,7 @@ async fn sim_ensure_destination_propolis_undo( // needed. match osagactx .nexus() - .instance_ensure_unregistered(&opctx, &authz_instance, &dst_sled_id) + .instance_ensure_unregistered(&dst_propolis_id, &dst_sled_id) .await { Ok(_) => Ok(()), @@ -500,12 +490,6 @@ async fn sim_instance_migrate( let src_propolis_id = db_instance.runtime().propolis_id.unwrap(); let dst_vmm = sagactx.lookup::("dst_vmm_record")?; - let (.., authz_instance) = LookupPath::new(&opctx, &osagactx.datastore()) - .instance_id(db_instance.id()) - .lookup_for(authz::Action::Modify) - .await - .map_err(ActionError::action_failed)?; - info!(osagactx.log(), "initiating migration from destination sled"; "instance_id" => %db_instance.id(), "dst_vmm_record" => ?dst_vmm, @@ -529,7 +513,6 @@ async fn sim_instance_migrate( .nexus() .instance_request_state( &opctx, - &authz_instance, &db_instance, &Some(dst_vmm), InstanceStateChangeRequest::Migrate( diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index 55fc312ae7..b6b78bd43c 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -538,6 +538,7 @@ async fn sis_ensure_registered_undo( let params = sagactx.saga_params::()?; let datastore = osagactx.datastore(); let instance_id = InstanceUuid::from_untyped_uuid(params.db_instance.id()); + let propolis_id = sagactx.lookup::("propolis_id")?; let sled_id = sagactx.lookup::("sled_id")?; let opctx = crate::context::op_context_for_saga_action( &sagactx, @@ -546,11 +547,12 @@ async fn sis_ensure_registered_undo( info!(osagactx.log(), "start saga: unregistering instance from sled"; "instance_id" => %instance_id, + "propolis_id" => %propolis_id, "sled_id" => %sled_id); // Fetch the latest record so that this callee can drive the instance into // a Failed state if the unregister call fails. - let (.., authz_instance, db_instance) = LookupPath::new(&opctx, &datastore) + let (.., db_instance) = LookupPath::new(&opctx, &datastore) .instance_id(instance_id.into_untyped_uuid()) .fetch() .await @@ -563,7 +565,7 @@ async fn sis_ensure_registered_undo( // returned. if let Err(e) = osagactx .nexus() - .instance_ensure_unregistered(&opctx, &authz_instance, &sled_id) + .instance_ensure_unregistered(&propolis_id, &sled_id) .await { error!(osagactx.log(), @@ -644,7 +646,6 @@ async fn sis_ensure_running( ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; - let datastore = osagactx.datastore(); let opctx = crate::context::op_context_for_saga_action( &sagactx, ¶ms.serialized_authn, @@ -659,17 +660,10 @@ async fn sis_ensure_running( "instance_id" => %instance_id, "sled_id" => %sled_id); - let (.., authz_instance) = LookupPath::new(&opctx, &datastore) - .instance_id(instance_id.into_untyped_uuid()) - .lookup_for(authz::Action::Modify) - .await - .map_err(ActionError::action_failed)?; - match osagactx .nexus() .instance_request_state( &opctx, - &authz_instance, &db_instance, &Some(db_vmm), crate::app::instance::InstanceStateChangeRequest::Run, diff --git a/nexus/src/app/sagas/instance_update/mod.rs b/nexus/src/app/sagas/instance_update/mod.rs index 5f226480b8..4c4c4deff2 100644 --- a/nexus/src/app/sagas/instance_update/mod.rs +++ b/nexus/src/app/sagas/instance_update/mod.rs @@ -30,10 +30,9 @@ //! Nexus' `cpapi_instances_put` internal API endpoint, when a Nexus' //! `instance-watcher` background task *pulls* instance states from sled-agents //! periodically, or as the return value of an API call from Nexus to a -//! sled-agent. When a Nexus receives a new [`SledInstanceState`] from a -//! sled-agent through any of these mechanisms, the Nexus will write any changed -//! state to the `vmm` and/or `migration` tables directly on behalf of the -//! sled-agent. +//! sled-agent. When a Nexus receives a new [`SledVmmState`] from a sled-agent +//! through any of these mechanisms, the Nexus will write any changed state to +//! the `vmm` and/or `migration` tables directly on behalf of the sled-agent. //! //! Although Nexus is technically the party responsible for the database query //! that writes VMM and migration state updates received from sled-agent, it is @@ -236,9 +235,9 @@ //! updates is perhaps the simplest one: _avoiding unnecessary update sagas_. //! The `cpapi_instances_put` API endpoint and instance-watcher background tasks //! handle changes to VMM and migration states by calling the -//! [`notify_instance_updated`] method, which writes the new states to the -//! database and (potentially) starts an update saga. Naively, this method would -//! *always* start an update saga, but remember that --- as we discussed +//! [`process_vmm_update`] method, which writes the new states to the database +//! and (potentially) starts an update saga. Naively, this method would *always* +//! start an update saga, but remember that --- as we discussed //! [above](#background) --- many VMM/migration state changes don't actually //! require modifying the instance record. For example, if an instance's VMM //! transitions from [`VmmState::Starting`] to [`VmmState::Running`], that @@ -271,7 +270,7 @@ //! delayed. To improve the timeliness of update sagas, we will also explicitly //! activate the background task at any point where we know that an update saga //! *should* run but we were not able to run it. If an update saga cannot be -//! started, whether by [`notify_instance_updated`], a `start-instance-update` +//! started, whether by [`notify_vmm_updated`], a `start-instance-update` //! saga attempting to start its real saga, or an `instance-update` saga //! chaining into a new one as its last action, the `instance-watcher` //! background task is activated. Similarly, when a `start-instance-update` saga @@ -326,7 +325,8 @@ //! crate::app::db::datastore::DataStore::instance_updater_inherit_lock //! [instance_updater_unlock]: //! crate::app::db::datastore::DataStore::instance_updater_unlock -//! [`notify_instance_updated`]: crate::app::Nexus::notify_instance_updated +//! [`notify_vmm_updated`]: crate::app::Nexus::notify_vmm_updated +//! [`process_vmm_update`]: crate::app::instance::process_vmm_update //! //! [dist-locking]: //! https://martin.kleppmann.com/2016/02/08/how-to-do-distributed-locking.html @@ -362,7 +362,7 @@ use nexus_db_queries::{authn, authz}; use nexus_types::identity::Resource; use omicron_common::api::external::Error; use omicron_common::api::internal::nexus; -use omicron_common::api::internal::nexus::SledInstanceState; +use omicron_common::api::internal::nexus::SledVmmState; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::PropolisUuid; @@ -388,8 +388,8 @@ pub(crate) use self::start::{Params, SagaInstanceUpdate}; mod destroyed; /// Returns `true` if an `instance-update` saga should be executed as a result -/// of writing the provided [`SledInstanceState`] to the database with the -/// provided [`VmmStateUpdateResult`]. +/// of writing the provided [`SledVmmState`] to the database with the provided +/// [`VmmStateUpdateResult`]. /// /// We determine this only after actually updating the database records, /// because we don't know whether a particular VMM or migration state is @@ -407,8 +407,8 @@ mod destroyed; /// VMM/migration states. pub fn update_saga_needed( log: &slog::Logger, - instance_id: InstanceUuid, - state: &SledInstanceState, + propolis_id: PropolisUuid, + state: &SledVmmState, result: &VmmStateUpdateResult, ) -> bool { // Currently, an instance-update saga is required if (and only if): @@ -443,8 +443,7 @@ pub fn update_saga_needed( debug!(log, "new VMM runtime state from sled agent requires an \ instance-update saga"; - "instance_id" => %instance_id, - "propolis_id" => %state.propolis_id, + "propolis_id" => %propolis_id, "vmm_needs_update" => vmm_needs_update, "migration_in_needs_update" => migration_in_needs_update, "migration_out_needs_update" => migration_out_needs_update, diff --git a/nexus/src/app/sagas/region_replacement_start.rs b/nexus/src/app/sagas/region_replacement_start.rs index 86aab2ac22..1bc1491468 100644 --- a/nexus/src/app/sagas/region_replacement_start.rs +++ b/nexus/src/app/sagas/region_replacement_start.rs @@ -747,7 +747,6 @@ pub(crate) mod test { }; use chrono::Utc; use nexus_db_model::Dataset; - use nexus_db_model::DatasetKind; use nexus_db_model::Region; use nexus_db_model::RegionReplacement; use nexus_db_model::RegionReplacementState; @@ -758,6 +757,7 @@ pub(crate) mod test { use nexus_test_utils::resource_helpers::create_project; use nexus_test_utils_macros::nexus_test; use nexus_types::identity::Asset; + use omicron_common::api::internal::shared::DatasetKind; use sled_agent_client::types::VolumeConstructionRequest; use uuid::Uuid; diff --git a/nexus/src/app/sagas/snapshot_create.rs b/nexus/src/app/sagas/snapshot_create.rs index eeb14091b2..540ab90e28 100644 --- a/nexus/src/app/sagas/snapshot_create.rs +++ b/nexus/src/app/sagas/snapshot_create.rs @@ -106,11 +106,12 @@ use nexus_db_queries::db::lookup::LookupPath; use omicron_common::api::external; use omicron_common::api::external::Error; use omicron_common::retry_until_known_result; +use omicron_uuid_kinds::{GenericUuid, PropolisUuid, SledUuid}; use rand::{rngs::StdRng, RngCore, SeedableRng}; use serde::Deserialize; use serde::Serialize; use sled_agent_client::types::CrucibleOpts; -use sled_agent_client::types::InstanceIssueDiskSnapshotRequestBody; +use sled_agent_client::types::VmmIssueDiskSnapshotRequestBody; use sled_agent_client::types::VolumeConstructionRequest; use slog::info; use std::collections::BTreeMap; @@ -826,39 +827,43 @@ async fn ssc_send_snapshot_request_to_sled_agent( .await .map_err(ActionError::action_failed)?; - let sled_id = osagactx + let instance_and_vmm = osagactx .datastore() .instance_fetch_with_vmm(&opctx, &authz_instance) .await - .map_err(ActionError::action_failed)? - .sled_id(); + .map_err(ActionError::action_failed)?; + + let vmm = instance_and_vmm.vmm(); // If this instance does not currently have a sled, we can't continue this // saga - the user will have to reissue the snapshot request and it will get // run on a Pantry. - let Some(sled_id) = sled_id else { + let Some((propolis_id, sled_id)) = + vmm.as_ref().map(|vmm| (vmm.id, vmm.sled_id)) + else { return Err(ActionError::action_failed(Error::unavail( - "sled id is None!", + "instance no longer has an active VMM!", ))); }; info!(log, "asking for disk snapshot from Propolis via sled agent"; "disk_id" => %params.disk_id, "instance_id" => %attach_instance_id, + "propolis_id" => %propolis_id, "sled_id" => %sled_id); let sled_agent_client = osagactx .nexus() - .sled_client(&sled_id) + .sled_client(&SledUuid::from_untyped_uuid(sled_id)) .await .map_err(ActionError::action_failed)?; retry_until_known_result(log, || async { sled_agent_client - .instance_issue_disk_snapshot_request( - &attach_instance_id, + .vmm_issue_disk_snapshot_request( + &PropolisUuid::from_untyped_uuid(propolis_id), ¶ms.disk_id, - &InstanceIssueDiskSnapshotRequestBody { snapshot_id }, + &VmmIssueDiskSnapshotRequestBody { snapshot_id }, ) .await }) @@ -2151,12 +2156,15 @@ mod test { .await .unwrap(); - let sled_id = instance_state - .sled_id() - .expect("starting instance should have a sled"); + let vmm_state = instance_state + .vmm() + .as_ref() + .expect("starting instance should have a vmm"); + let propolis_id = PropolisUuid::from_untyped_uuid(vmm_state.id); + let sled_id = SledUuid::from_untyped_uuid(vmm_state.sled_id); let sa = nexus.sled_client(&sled_id).await.unwrap(); + sa.vmm_finish_transition(propolis_id).await; - sa.instance_finish_transition(instance.identity.id).await; let instance_state = nexus .datastore() .instance_fetch_with_vmm(&opctx, &authz_instance) diff --git a/nexus/src/app/sagas/test_helpers.rs b/nexus/src/app/sagas/test_helpers.rs index b9388a1116..1572ba4330 100644 --- a/nexus/src/app/sagas/test_helpers.rs +++ b/nexus/src/app/sagas/test_helpers.rs @@ -5,11 +5,8 @@ //! Helper functions for writing saga undo tests and working with instances in //! saga tests. -use super::NexusSaga; -use crate::{ - app::{saga::create_saga_dag, test_interfaces::TestInterfaces as _}, - Nexus, -}; +use super::{instance_common::VmmAndSledIds, NexusSaga}; +use crate::{app::saga::create_saga_dag, Nexus}; use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use camino::Utf8Path; use diesel::{ @@ -137,13 +134,14 @@ pub(crate) async fn instance_simulate( info!(&cptestctx.logctx.log, "Poking simulated instance"; "instance_id" => %instance_id); let nexus = &cptestctx.server.server_context().nexus; + let VmmAndSledIds { vmm_id, sled_id } = + instance_fetch_vmm_and_sled_ids(cptestctx, instance_id).await; let sa = nexus - .instance_sled_by_id(instance_id) + .sled_client(&sled_id) .await - .unwrap() .expect("instance must be on a sled to simulate a state change"); - sa.instance_finish_transition(instance_id.into_untyped_uuid()).await; + sa.vmm_finish_transition(vmm_id).await; } pub(crate) async fn instance_single_step_on_sled( @@ -158,12 +156,14 @@ pub(crate) async fn instance_single_step_on_sled( "sled_id" => %sled_id, ); let nexus = &cptestctx.server.server_context().nexus; + let VmmAndSledIds { vmm_id, sled_id } = + instance_fetch_vmm_and_sled_ids(cptestctx, instance_id).await; let sa = nexus - .sled_client(sled_id) + .sled_client(&sled_id) .await - .expect("sled must exist to simulate a state change"); + .expect("instance must be on a sled to simulate a state change"); - sa.instance_single_step(instance_id.into_untyped_uuid()).await; + sa.vmm_single_step(vmm_id).await; } pub(crate) async fn instance_simulate_by_name( @@ -186,12 +186,14 @@ pub(crate) async fn instance_simulate_by_name( let instance_lookup = nexus.instance_lookup(&opctx, instance_selector).unwrap(); let (.., instance) = instance_lookup.fetch().await.unwrap(); + let instance_id = InstanceUuid::from_untyped_uuid(instance.id()); + let VmmAndSledIds { vmm_id, sled_id } = + instance_fetch_vmm_and_sled_ids(cptestctx, &instance_id).await; let sa = nexus - .instance_sled_by_id(&InstanceUuid::from_untyped_uuid(instance.id())) + .sled_client(&sled_id) .await - .unwrap() .expect("instance must be on a sled to simulate a state change"); - sa.instance_finish_transition(instance.id()).await; + sa.vmm_finish_transition(vmm_id).await; } pub async fn instance_fetch( @@ -218,6 +220,21 @@ pub async fn instance_fetch( db_state } +pub(super) async fn instance_fetch_vmm_and_sled_ids( + cptestctx: &ControlPlaneTestContext, + instance_id: &InstanceUuid, +) -> VmmAndSledIds { + let instance_and_vmm = instance_fetch(cptestctx, *instance_id).await; + let vmm = instance_and_vmm + .vmm() + .as_ref() + .expect("can only fetch VMM and sled IDs for an active instance"); + + let vmm_id = PropolisUuid::from_untyped_uuid(vmm.id); + let sled_id = SledUuid::from_untyped_uuid(vmm.sled_id); + VmmAndSledIds { vmm_id, sled_id } +} + pub async fn instance_fetch_all( cptestctx: &ControlPlaneTestContext, instance_id: InstanceUuid, diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index 261045670e..9c21ca73a1 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -12,7 +12,6 @@ use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use nexus_db_queries::db::lookup; -use nexus_db_queries::db::model::DatasetKind; use nexus_sled_agent_shared::inventory::SledRole; use nexus_types::deployment::DiskFilter; use nexus_types::deployment::SledFilter; @@ -23,6 +22,7 @@ use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; +use omicron_common::api::internal::shared::DatasetKind; use omicron_uuid_kinds::{GenericUuid, SledUuid}; use sled_agent_client::Client as SledAgentClient; use std::net::SocketAddrV6; @@ -292,13 +292,12 @@ impl super::Nexus { // Datasets (contained within zpools) - /// Upserts a dataset into the database, updating it if it already exists. - pub(crate) async fn upsert_dataset( + /// Upserts a crucible dataset into the database, updating it if it already exists. + pub(crate) async fn upsert_crucible_dataset( &self, id: Uuid, zpool_id: Uuid, address: SocketAddrV6, - kind: DatasetKind, ) -> Result<(), Error> { info!( self.log, @@ -307,6 +306,7 @@ impl super::Nexus { "dataset_id" => id.to_string(), "address" => address.to_string() ); + let kind = DatasetKind::Crucible; let dataset = db::model::Dataset::new(id, zpool_id, Some(address), kind); self.db_datastore.dataset_upsert(dataset).await?; diff --git a/nexus/src/app/snapshot.rs b/nexus/src/app/snapshot.rs index 040c9fc082..57b8edd1f0 100644 --- a/nexus/src/app/snapshot.rs +++ b/nexus/src/app/snapshot.rs @@ -109,7 +109,7 @@ impl super::Nexus { // If a Propolis _may_ exist, send the snapshot request there, // otherwise use the pantry. - !instance_state.vmm().is_some() + instance_state.vmm().is_none() } else { // This disk is not attached to an instance, use the pantry. true diff --git a/nexus/src/app/test_interfaces.rs b/nexus/src/app/test_interfaces.rs index adfafa523d..9852225e8c 100644 --- a/nexus/src/app/test_interfaces.rs +++ b/nexus/src/app/test_interfaces.rs @@ -6,8 +6,7 @@ use async_trait::async_trait; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::lookup::LookupPath; use omicron_common::api::external::Error; -use omicron_uuid_kinds::GenericUuid; -use omicron_uuid_kinds::{InstanceUuid, SledUuid}; +use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid, SledUuid}; use sled_agent_client::Client as SledAgentClient; use std::sync::Arc; use uuid::Uuid; @@ -19,25 +18,47 @@ pub use super::update::SpUpdater; pub use super::update::UpdateProgress; pub use gateway_client::types::SpType; +/// The information needed to talk to a sled agent about an instance that is +/// active on that sled. +pub struct InstanceSledAgentInfo { + /// The ID of the Propolis job to send to sled agent. + pub propolis_id: PropolisUuid, + + /// The ID of the sled where the Propolis job is running. + pub sled_id: SledUuid, + + /// A client for talking to the Propolis's host sled. + pub sled_client: Arc, + + /// The ID of the instance's migration target Propolis, if it has one. + pub dst_propolis_id: Option, +} + /// Exposes additional [`super::Nexus`] interfaces for use by the test suite #[async_trait] pub trait TestInterfaces { /// Access the Rack ID of the currently executing Nexus. fn rack_id(&self) -> Uuid; - /// Returns the SledAgentClient for an Instance from its id. We may also - /// want to split this up into instance_lookup_by_id() and instance_sled(), - /// but after all it's a test suite special to begin with. - async fn instance_sled_by_id( + /// Attempts to obtain the Propolis ID and sled agent information for an + /// instance. + /// + /// # Arguments + /// + /// - `id`: The ID of the instance of interest. + /// - `opctx`: An optional operation context to use for authorization + /// checks. If `None`, this routine supplies the default test opctx. + /// + /// # Return value + /// + /// - `Ok(Some(info))` if the instance has an active Propolis. + /// - `Ok(None)` if the instance has no active Propolis. + /// - `Err` if an error occurred. + async fn active_instance_info( &self, id: &InstanceUuid, - ) -> Result>, Error>; - - async fn instance_sled_by_id_with_opctx( - &self, - id: &InstanceUuid, - opctx: &OpContext, - ) -> Result>, Error>; + opctx: Option<&OpContext>, + ) -> Result, Error>; /// Returns the SledAgentClient for the sled running an instance to which a /// disk is attached. @@ -46,18 +67,6 @@ pub trait TestInterfaces { id: &Uuid, ) -> Result>, Error>; - /// Returns the supplied instance's current active sled ID. - async fn instance_sled_id( - &self, - instance_id: &InstanceUuid, - ) -> Result, Error>; - - async fn instance_sled_id_with_opctx( - &self, - instance_id: &InstanceUuid, - opctx: &OpContext, - ) -> Result, Error>; - async fn set_disk_as_faulted(&self, disk_id: &Uuid) -> Result; fn set_samael_max_issue_delay(&self, max_issue_delay: chrono::Duration); @@ -69,30 +78,49 @@ impl TestInterfaces for super::Nexus { self.rack_id } - async fn instance_sled_by_id( + async fn active_instance_info( &self, id: &InstanceUuid, - ) -> Result>, Error> { - let opctx = OpContext::for_tests( - self.log.new(o!()), - Arc::clone(&self.db_datastore) - as Arc, - ); + opctx: Option<&OpContext>, + ) -> Result, Error> { + let local_opctx; + let opctx = match opctx { + Some(o) => o, + None => { + local_opctx = OpContext::for_tests( + self.log.new(o!()), + Arc::clone(&self.db_datastore) + as Arc, + ); + &local_opctx + } + }; - self.instance_sled_by_id_with_opctx(id, &opctx).await - } + let (.., authz_instance) = LookupPath::new(&opctx, &self.db_datastore) + .instance_id(id.into_untyped_uuid()) + .lookup_for(nexus_db_queries::authz::Action::Read) + .await?; - async fn instance_sled_by_id_with_opctx( - &self, - id: &InstanceUuid, - opctx: &OpContext, - ) -> Result>, Error> { - let sled_id = self.instance_sled_id_with_opctx(id, opctx).await?; - if let Some(sled_id) = sled_id { - Ok(Some(self.sled_client(&sled_id).await?)) - } else { - Ok(None) - } + let state = self + .datastore() + .instance_fetch_with_vmm(opctx, &authz_instance) + .await?; + + let Some(vmm) = state.vmm() else { + return Ok(None); + }; + + let sled_id = SledUuid::from_untyped_uuid(vmm.sled_id); + Ok(Some(InstanceSledAgentInfo { + propolis_id: PropolisUuid::from_untyped_uuid(vmm.id), + sled_id, + sled_client: self.sled_client(&sled_id).await?, + dst_propolis_id: state + .instance() + .runtime() + .dst_propolis_id + .map(PropolisUuid::from_untyped_uuid), + })) } async fn disk_sled_by_id( @@ -112,37 +140,11 @@ impl TestInterfaces for super::Nexus { let instance_id = InstanceUuid::from_untyped_uuid( db_disk.runtime().attach_instance_id.unwrap(), ); - self.instance_sled_by_id(&instance_id).await - } - - async fn instance_sled_id( - &self, - id: &InstanceUuid, - ) -> Result, Error> { - let opctx = OpContext::for_tests( - self.log.new(o!()), - Arc::clone(&self.db_datastore) - as Arc, - ); - - self.instance_sled_id_with_opctx(id, &opctx).await - } - - async fn instance_sled_id_with_opctx( - &self, - id: &InstanceUuid, - opctx: &OpContext, - ) -> Result, Error> { - let (.., authz_instance) = LookupPath::new(&opctx, &self.db_datastore) - .instance_id(id.into_untyped_uuid()) - .lookup_for(nexus_db_queries::authz::Action::Read) - .await?; Ok(self - .datastore() - .instance_fetch_with_vmm(opctx, &authz_instance) + .active_instance_info(&instance_id, Some(&opctx)) .await? - .sled_id()) + .map(|info| info.sled_client)) } async fn set_disk_as_faulted(&self, disk_id: &Uuid) -> Result { diff --git a/nexus/src/bin/nexus.rs b/nexus/src/bin/nexus.rs index 33870b39e3..01e4bfc3af 100644 --- a/nexus/src/bin/nexus.rs +++ b/nexus/src/bin/nexus.rs @@ -16,20 +16,11 @@ use clap::Parser; use nexus_config::NexusConfig; use omicron_common::cmd::fatal; use omicron_common::cmd::CmdError; -use omicron_nexus::run_openapi_external; use omicron_nexus::run_server; #[derive(Debug, Parser)] #[clap(name = "nexus", about = "See README.adoc for more information")] struct Args { - #[clap( - short = 'O', - long = "openapi", - help = "Print the external OpenAPI Spec document and exit", - action - )] - openapi: bool, - #[clap(name = "CONFIG_FILE_PATH", action)] config_file_path: Option, } @@ -44,23 +35,19 @@ async fn main() { async fn do_run() -> Result<(), CmdError> { let args = Args::parse(); - if args.openapi { - run_openapi_external().map_err(|err| CmdError::Failure(anyhow!(err))) - } else { - let config_path = match args.config_file_path { - Some(path) => path, - None => { - use clap::CommandFactory; - - eprintln!("{}", Args::command().render_help()); - return Err(CmdError::Usage( - "CONFIG_FILE_PATH is required".to_string(), - )); - } - }; - let config = NexusConfig::from_file(config_path) - .map_err(|e| CmdError::Failure(anyhow!(e)))?; - - run_server(&config).await.map_err(|err| CmdError::Failure(anyhow!(err))) - } + let config_path = match args.config_file_path { + Some(path) => path, + None => { + use clap::CommandFactory; + + eprintln!("{}", Args::command().render_help()); + return Err(CmdError::Usage( + "CONFIG_FILE_PATH is required".to_string(), + )); + } + }; + let config = NexusConfig::from_file(config_path) + .map_err(|e| CmdError::Failure(anyhow!(e)))?; + + run_server(&config).await.map_err(|err| CmdError::Failure(anyhow!(err))) } diff --git a/nexus/src/bin/schema-updater.rs b/nexus/src/bin/schema-updater.rs index 7fe1ed84a4..4a43698f00 100644 --- a/nexus/src/bin/schema-updater.rs +++ b/nexus/src/bin/schema-updater.rs @@ -71,7 +71,7 @@ async fn main() -> anyhow::Result<()> { let log = Logger::root(drain, slog::o!("unit" => "schema_updater")); let crdb_cfg = db::Config { url: args.url }; - let pool = Arc::new(db::Pool::new(&log, &crdb_cfg)); + let pool = Arc::new(db::Pool::new_single_host(&log, &crdb_cfg)); let schema_config = SchemaConfig { schema_dir: args.schema_directory }; let all_versions = AllSchemaVersions::load(&schema_config.schema_dir)?; diff --git a/nexus/src/context.rs b/nexus/src/context.rs index 95d69e0c88..9620a3937a 100644 --- a/nexus/src/context.rs +++ b/nexus/src/context.rs @@ -11,9 +11,7 @@ use authn::external::token::HttpAuthnToken; use authn::external::HttpAuthnScheme; use camino::Utf8PathBuf; use chrono::Duration; -use internal_dns::ServiceName; use nexus_config::NexusConfig; -use nexus_config::PostgresConfigWithUrl; use nexus_config::SchemeName; use nexus_db_queries::authn::external::session_cookie::SessionStore; use nexus_db_queries::authn::ConsoleSessionWithSiloId; @@ -25,7 +23,6 @@ use oximeter::types::ProducerRegistry; use oximeter_instruments::http::{HttpService, LatencyTracker}; use slog::Logger; use std::env; -use std::str::FromStr; use std::sync::Arc; use uuid::Uuid; @@ -149,12 +146,14 @@ impl ServerContext { name: name.to_string().into(), id: config.deployment.id, }; - const START_LATENCY_DECADE: i16 = -6; - const END_LATENCY_DECADE: i16 = 3; - LatencyTracker::with_latency_decades( + // Start at 1 microsecond == 1e3 nanoseconds. + const LATENCY_START_POWER: u16 = 3; + // End at 1000s == (1e9 * 1e3) == 1e12 nanoseconds. + const LATENCY_END_POWER: u16 = 12; + LatencyTracker::with_log_linear_bins( target, - START_LATENCY_DECADE, - END_LATENCY_DECADE, + LATENCY_START_POWER, + LATENCY_END_POWER, ) .unwrap() }; @@ -210,7 +209,7 @@ impl ServerContext { // nexus in dev for everyone // Set up DNS Client - let resolver = match config.deployment.internal_dns { + let (resolver, dns_addrs) = match config.deployment.internal_dns { nexus_config::InternalDns::FromSubnet { subnet } => { let az_subnet = Ipv6Subnet::::new(subnet.net().addr()); @@ -219,11 +218,21 @@ impl ServerContext { "Setting up resolver using DNS servers for subnet: {:?}", az_subnet ); - internal_dns::resolver::Resolver::new_from_subnet( - log.new(o!("component" => "DnsResolver")), - az_subnet, + let resolver = + internal_dns::resolver::Resolver::new_from_subnet( + log.new(o!("component" => "DnsResolver")), + az_subnet, + ) + .map_err(|e| { + format!("Failed to create DNS resolver: {}", e) + })?; + + ( + resolver, + internal_dns::resolver::Resolver::servers_from_subnet( + az_subnet, + ), ) - .map_err(|e| format!("Failed to create DNS resolver: {}", e))? } nexus_config::InternalDns::FromAddress { address } => { info!( @@ -231,56 +240,33 @@ impl ServerContext { "Setting up resolver using DNS address: {:?}", address ); - internal_dns::resolver::Resolver::new_from_addrs( - log.new(o!("component" => "DnsResolver")), - &[address], - ) - .map_err(|e| format!("Failed to create DNS resolver: {}", e))? + let resolver = + internal_dns::resolver::Resolver::new_from_addrs( + log.new(o!("component" => "DnsResolver")), + &[address], + ) + .map_err(|e| { + format!("Failed to create DNS resolver: {}", e) + })?; + + (resolver, vec![address]) } }; - // Set up DB pool - let url = match &config.deployment.database { - nexus_config::Database::FromUrl { url } => url.clone(), + let pool = match &config.deployment.database { + nexus_config::Database::FromUrl { url } => { + info!(log, "Setting up qorb pool from a single host"; "url" => #?url); + db::Pool::new_single_host( + &log, + &db::Config { url: url.clone() }, + ) + } nexus_config::Database::FromDns => { - info!(log, "Accessing DB url from DNS"); - // It's been requested but unfortunately not supported to - // directly connect using SRV based lookup. - // TODO-robustness: the set of cockroachdb hosts we'll use will - // be fixed to whatever we got back from DNS at Nexus start. - // This means a new cockroachdb instance won't picked up until - // Nexus restarts. - let addrs = loop { - match resolver - .lookup_all_socket_v6(ServiceName::Cockroach) - .await - { - Ok(addrs) => break addrs, - Err(e) => { - warn!( - log, - "Failed to lookup cockroach addresses: {e}" - ); - tokio::time::sleep(std::time::Duration::from_secs( - 1, - )) - .await; - } - } - }; - let addrs_str = addrs - .iter() - .map(ToString::to_string) - .collect::>() - .join(","); - info!(log, "DB addresses: {}", addrs_str); - PostgresConfigWithUrl::from_str(&format!( - "postgresql://root@{addrs_str}/omicron?sslmode=disable", - )) - .map_err(|e| format!("Cannot parse Postgres URL: {}", e))? + info!(log, "Setting up qorb pool from DNS"; "dns_addrs" => #?dns_addrs); + db::Pool::new(&log, dns_addrs) } }; - let pool = db::Pool::new(&log, &db::Config { url }); + let nexus = Nexus::new_with_id( rack_id, log.new(o!("component" => "nexus")), diff --git a/nexus/src/external_api/console_api.rs b/nexus/src/external_api/console_api.rs index 2169b631a7..4ea8290bf9 100644 --- a/nexus/src/external_api/console_api.rs +++ b/nexus/src/external_api/console_api.rs @@ -25,11 +25,11 @@ use crate::context::ApiContext; use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; use dropshot::{ - endpoint, http_response_found, http_response_see_other, HttpError, - HttpResponseFound, HttpResponseHeaders, HttpResponseSeeOther, - HttpResponseUpdatedNoContent, Path, Query, RequestContext, + http_response_found, http_response_see_other, HttpError, HttpResponseFound, + HttpResponseHeaders, HttpResponseSeeOther, HttpResponseUpdatedNoContent, + Path, Query, RequestContext, }; -use http::{header, HeaderName, HeaderValue, Response, StatusCode, Uri}; +use http::{header, HeaderName, HeaderValue, Response, StatusCode}; use hyper::Body; use nexus_db_model::AuthenticationMode; use nexus_db_queries::authn::silos::IdentityProviderType; @@ -42,18 +42,16 @@ use nexus_db_queries::{ db::identity::Asset, }; use nexus_types::authn::cookies::Cookies; -use nexus_types::external_api::params; +use nexus_types::external_api::params::{self, RelativeUri}; use nexus_types::identity::Resource; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::{DataPageParams, Error, NameOrId}; use once_cell::sync::Lazy; -use parse_display::Display; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_urlencoded; use std::collections::HashMap; use std::num::NonZeroU32; -use std::str::FromStr; use tokio::fs::File; use tokio_util::codec::{BytesCodec, FramedRead}; @@ -194,12 +192,6 @@ use tokio_util::codec::{BytesCodec, FramedRead}; // // /logout/{silo_name}/{provider_name} -#[derive(Deserialize, JsonSchema)] -pub struct LoginToProviderPathParam { - pub silo_name: nexus_db_queries::db::model::Name, - pub provider_name: nexus_db_queries::db::model::Name, -} - #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct RelayState { pub redirect_uri: Option, @@ -228,36 +220,18 @@ impl RelayState { } } -/// SAML login console page (just a link to the IdP) -#[endpoint { - method = GET, - path = "/login/{silo_name}/saml/{provider_name}", - tags = ["login"], - unpublished = true, -}] pub(crate) async fn login_saml_begin( rqctx: RequestContext, - _path_params: Path, - _query_params: Query, + _path_params: Path, + _query_params: Query, ) -> Result, HttpError> { serve_console_index(rqctx).await } -/// Get a redirect straight to the IdP -/// -/// Console uses this to avoid having to ask the API anything about the IdP. It -/// already knows the IdP name from the path, so it can just link to this path -/// and rely on Nexus to redirect to the actual IdP. -#[endpoint { - method = GET, - path = "/login/{silo_name}/saml/{provider_name}/redirect", - tags = ["login"], - unpublished = true, -}] pub(crate) async fn login_saml_redirect( rqctx: RequestContext, - path_params: Path, - query_params: Query, + path_params: Path, + query_params: Query, ) -> Result { let apictx = rqctx.context(); let handler = async { @@ -272,8 +246,8 @@ pub(crate) async fn login_saml_redirect( .datastore() .identity_provider_lookup( &opctx, - &path_params.silo_name, - &path_params.provider_name, + &path_params.silo_name.into(), + &path_params.provider_name.into(), ) .await?; @@ -308,15 +282,9 @@ pub(crate) async fn login_saml_redirect( .await } -/// Authenticate a user via SAML -#[endpoint { - method = POST, - path = "/login/{silo_name}/saml/{provider_name}", - tags = ["login"], -}] pub(crate) async fn login_saml( rqctx: RequestContext, - path_params: Path, + path_params: Path, body_bytes: dropshot::UntypedBody, ) -> Result { let apictx = rqctx.context(); @@ -333,8 +301,8 @@ pub(crate) async fn login_saml( .datastore() .identity_provider_lookup( &opctx, - &path_params.silo_name, - &path_params.provider_name, + &path_params.silo_name.into(), + &path_params.provider_name.into(), ) .await?; @@ -395,21 +363,10 @@ pub(crate) async fn login_saml( .await } -#[derive(Deserialize, JsonSchema)] -pub struct LoginPathParam { - pub silo_name: nexus_db_queries::db::model::Name, -} - -#[endpoint { - method = GET, - path = "/login/{silo_name}/local", - tags = ["login"], - unpublished = true, -}] pub(crate) async fn login_local_begin( rqctx: RequestContext, - _path_params: Path, - _query_params: Query, + _path_params: Path, + _query_params: Query, ) -> Result, HttpError> { // TODO: figure out why instrumenting doesn't work // let apictx = rqctx.context(); @@ -418,15 +375,9 @@ pub(crate) async fn login_local_begin( serve_console_index(rqctx).await } -/// Authenticate a user via username and password -#[endpoint { - method = POST, - path = "/v1/login/{silo_name}/local", - tags = ["login"], -}] pub(crate) async fn login_local( rqctx: RequestContext, - path_params: Path, + path_params: Path, credentials: dropshot::TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); @@ -485,13 +436,6 @@ async fn create_session( Ok(session) } -/// Log user out of web console by deleting session on client and server -#[endpoint { - // important for security that this be a POST despite the empty req body - method = POST, - path = "/v1/logout", - tags = ["hidden"], -}] pub(crate) async fn logout( rqctx: RequestContext, cookies: Cookies, @@ -539,53 +483,6 @@ pub(crate) async fn logout( .await } -#[derive(Deserialize, JsonSchema)] -pub struct RestPathParam { - path: Vec, -} - -/// This is meant as a security feature. We want to ensure we never redirect to -/// a URI on a different host. -#[derive(Serialize, Deserialize, Debug, JsonSchema, Clone, Display)] -#[serde(try_from = "String")] -#[display("{0}")] -pub struct RelativeUri(String); - -impl FromStr for RelativeUri { - type Err = String; - - fn from_str(s: &str) -> Result { - Self::try_from(s.to_string()) - } -} - -impl TryFrom for RelativeUri { - type Error = String; - - fn try_from(uri: Uri) -> Result { - if uri.host().is_none() && uri.scheme().is_none() { - Ok(Self(uri.to_string())) - } else { - Err(format!("\"{}\" is not a relative URI", uri)) - } - } -} - -impl TryFrom for RelativeUri { - type Error = String; - - fn try_from(s: String) -> Result { - s.parse::() - .map_err(|_| format!("\"{}\" is not a relative URI", s)) - .and_then(|uri| Self::try_from(uri)) - } -} - -#[derive(Serialize, Deserialize, JsonSchema)] -pub struct LoginUrlQuery { - redirect_uri: Option, -} - /// Generate URI to the appropriate login form for this Silo. Optional /// `redirect_uri` represents the URL to send the user back to after successful /// login, and is included in `state` query param if present @@ -642,7 +539,7 @@ async fn get_login_url( // Stick redirect_url into the state param and URL encode it so it can be // used as a query string. We assume it's not already encoded. - let query_data = LoginUrlQuery { redirect_uri }; + let query_data = params::LoginUrlQuery { redirect_uri }; Ok(match serde_urlencoded::to_string(query_data) { // only put the ? in front if there's something there @@ -652,15 +549,9 @@ async fn get_login_url( }) } -/// Redirect to a login page for the current Silo (if that can be determined) -#[endpoint { - method = GET, - path = "/login", - unpublished = true, -}] pub(crate) async fn login_begin( rqctx: RequestContext, - query_params: Query, + query_params: Query, ) -> Result { let apictx = rqctx.context(); let handler = async { @@ -694,7 +585,11 @@ pub(crate) async fn console_index_or_login_redirect( .request .uri() .path_and_query() - .map(|p| RelativeUri(p.to_string())); + .map(|p| p.to_string().parse::()) + .transpose() + .map_err(|e| { + HttpError::for_internal_error(format!("parsing URI: {}", e)) + })?; let login_url = get_login_url(&rqctx, redirect_uri).await?; Ok(Response::builder() @@ -709,8 +604,7 @@ pub(crate) async fn console_index_or_login_redirect( // to manually define more specific routes. macro_rules! console_page { - ($name:ident, $path:literal) => { - #[endpoint { method = GET, path = $path, unpublished = true, }] + ($name:ident) => { pub(crate) async fn $name( rqctx: RequestContext, ) -> Result, HttpError> { @@ -721,26 +615,25 @@ macro_rules! console_page { // only difference is the _path_params arg macro_rules! console_page_wildcard { - ($name:ident, $path:literal) => { - #[endpoint { method = GET, path = $path, unpublished = true, }] + ($name:ident) => { pub(crate) async fn $name( rqctx: RequestContext, - _path_params: Path, + _path_params: Path, ) -> Result, HttpError> { console_index_or_login_redirect(rqctx).await } }; } -console_page_wildcard!(console_projects, "/projects/{path:.*}"); -console_page_wildcard!(console_settings_page, "/settings/{path:.*}"); -console_page_wildcard!(console_system_page, "/system/{path:.*}"); -console_page_wildcard!(console_lookup, "/lookup/{path:.*}"); -console_page!(console_root, "/"); -console_page!(console_projects_new, "/projects-new"); -console_page!(console_silo_images, "/images"); -console_page!(console_silo_utilization, "/utilization"); -console_page!(console_silo_access, "/access"); +console_page_wildcard!(console_projects); +console_page_wildcard!(console_settings_page); +console_page_wildcard!(console_system_page); +console_page_wildcard!(console_lookup); +console_page!(console_root); +console_page!(console_projects_new); +console_page!(console_silo_images); +console_page!(console_silo_utilization); +console_page!(console_silo_access); /// Check if `gzip` is listed in the request's `Accept-Encoding` header. fn accept_gz(header_value: &str) -> bool { @@ -868,15 +761,10 @@ async fn serve_static( /// /// Note that Dropshot protects us from directory traversal attacks (e.g. /// `/assets/../../../etc/passwd`). This is tested in the `console_api` -/// integration tests. -#[endpoint { - method = GET, - path = "/assets/{path:.*}", - unpublished = true, -}] +/// integration tests pub(crate) async fn asset( rqctx: RequestContext, - path_params: Path, + path_params: Path, ) -> Result, HttpError> { // asset URLs contain hashes, so cache for 1 year const CACHE_CONTROL: HeaderValue = diff --git a/nexus/src/external_api/device_auth.rs b/nexus/src/external_api/device_auth.rs index 883dbf4e19..87ccbd9752 100644 --- a/nexus/src/external_api/device_auth.rs +++ b/nexus/src/external_api/device_auth.rs @@ -14,16 +14,14 @@ use super::views::DeviceAccessTokenGrant; use crate::app::external_endpoints::authority_for_request; use crate::ApiContext; use dropshot::{ - endpoint, HttpError, HttpResponseUpdatedNoContent, RequestContext, - TypedBody, + HttpError, HttpResponseUpdatedNoContent, RequestContext, TypedBody, }; use http::{header, Response, StatusCode}; use hyper::Body; use nexus_db_queries::db::model::DeviceAccessToken; +use nexus_types::external_api::params; use omicron_common::api::external::InternalContext; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use uuid::Uuid; +use serde::Serialize; // Token granting à la RFC 8628 (OAuth 2.0 Device Authorization Grant) @@ -46,25 +44,9 @@ where .body(body.into())?) } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct DeviceAuthRequest { - pub client_id: Uuid, -} - -/// Start an OAuth 2.0 Device Authorization Grant -/// -/// This endpoint is designed to be accessed from an *unauthenticated* -/// API client. It generates and records a `device_code` and `user_code` -/// which must be verified and confirmed prior to a token being granted. -#[endpoint { - method = POST, - path = "/device/auth", - content_type = "application/x-www-form-urlencoded", - tags = ["hidden"], // "token" -}] pub(crate) async fn device_auth_request( rqctx: RequestContext, - params: TypedBody, + params: TypedBody, ) -> Result, HttpError> { let apictx = rqctx.context(); let nexus = &apictx.context.nexus; @@ -99,53 +81,21 @@ pub(crate) async fn device_auth_request( .await } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct DeviceAuthVerify { - pub user_code: String, -} - -/// Verify an OAuth 2.0 Device Authorization Grant -/// -/// This endpoint should be accessed in a full user agent (e.g., -/// a browser). If the user is not logged in, we redirect them to -/// the login page and use the `state` parameter to get them back -/// here on completion. If they are logged in, serve up the console -/// verification page so they can verify the user code. -#[endpoint { - method = GET, - path = "/device/verify", - unpublished = true, -}] pub(crate) async fn device_auth_verify( rqctx: RequestContext, ) -> Result, HttpError> { console_index_or_login_redirect(rqctx).await } -#[endpoint { - method = GET, - path = "/device/success", - unpublished = true, -}] pub(crate) async fn device_auth_success( rqctx: RequestContext, ) -> Result, HttpError> { console_index_or_login_redirect(rqctx).await } -/// Confirm an OAuth 2.0 Device Authorization Grant -/// -/// This endpoint is designed to be accessed by the user agent (browser), -/// not the client requesting the token. So we do not actually return the -/// token here; it will be returned in response to the poll on `/device/token`. -#[endpoint { - method = POST, - path = "/device/confirm", - tags = ["hidden"], // "token" -}] pub(crate) async fn device_auth_confirm( rqctx: RequestContext, - params: TypedBody, + params: TypedBody, ) -> Result { let apictx = rqctx.context(); let nexus = &apictx.context.nexus; @@ -171,13 +121,6 @@ pub(crate) async fn device_auth_confirm( .await } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct DeviceAccessTokenRequest { - pub grant_type: String, - pub device_code: String, - pub client_id: Uuid, -} - #[derive(Debug)] pub enum DeviceAccessTokenResponse { Granted(DeviceAccessToken), @@ -186,23 +129,12 @@ pub enum DeviceAccessTokenResponse { Denied, } -/// Request a device access token -/// -/// This endpoint should be polled by the client until the user code -/// is verified and the grant is confirmed. -#[endpoint { - method = POST, - path = "/device/token", - content_type = "application/x-www-form-urlencoded", - tags = ["hidden"], // "token" -}] pub(crate) async fn device_access_token( rqctx: RequestContext, - params: TypedBody, + params: params::DeviceAccessTokenRequest, ) -> Result, HttpError> { let apictx = rqctx.context(); let nexus = &apictx.context.nexus; - let params = params.into_inner(); let handler = async { // RFC 8628 §3.4 if params.grant_type != "urn:ietf:params:oauth:grant-type:device_code" { diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index e11256f06e..a297eaa533 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -14,8 +14,8 @@ use super::{ }, }; use crate::{context::ApiContext, external_api::shared}; -use dropshot::HttpResponseAccepted; -use dropshot::HttpResponseCreated; +use dropshot::EmptyScanParams; +use dropshot::HttpError; use dropshot::HttpResponseDeleted; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -27,12 +27,12 @@ use dropshot::RequestContext; use dropshot::ResultsPage; use dropshot::TypedBody; use dropshot::WhichPage; -use dropshot::{ - channel, endpoint, WebsocketChannelResult, WebsocketConnection, -}; use dropshot::{ApiDescription, StreamingBody}; -use dropshot::{ApiDescriptionRegisterError, HttpError}; -use dropshot::{ApiEndpoint, EmptyScanParams}; +use dropshot::{HttpResponseAccepted, HttpResponseFound, HttpResponseSeeOther}; +use dropshot::{HttpResponseCreated, HttpResponseHeaders}; +use dropshot::{WebsocketChannelResult, WebsocketConnection}; +use http::Response; +use hyper::Body; use ipnetwork::IpNetwork; use nexus_db_queries::authz; use nexus_db_queries::db; @@ -40,7 +40,14 @@ use nexus_db_queries::db::identity::Resource; use nexus_db_queries::db::lookup::ImageLookup; use nexus_db_queries::db::lookup::ImageParentLookup; use nexus_db_queries::db::model::Name; -use nexus_types::external_api::shared::{BfdStatus, ProbeInfo}; +use nexus_external_api::*; +use nexus_types::{ + authn::cookies::Cookies, + external_api::{ + params::SystemMetricsPathParam, + shared::{BfdStatus, ProbeInfo}, + }, +}; use omicron_common::api::external::http_pagination::data_page_params_for; use omicron_common::api::external::http_pagination::marker_for_name; use omicron_common::api::external::http_pagination::marker_for_name_or_id; @@ -83,7077 +90,5808 @@ use omicron_common::api::external::VpcFirewallRuleUpdateParams; use omicron_common::api::external::VpcFirewallRules; use omicron_common::bail_unless; use omicron_uuid_kinds::GenericUuid; -use parse_display::Display; use propolis_client::support::tungstenite::protocol::frame::coding::CloseCode; use propolis_client::support::tungstenite::protocol::{ CloseFrame, Role as WebSocketRole, }; use propolis_client::support::WebSocketStream; use ref_cast::RefCast; -use schemars::JsonSchema; -use serde::Deserialize; -use serde::Serialize; -use std::net::IpAddr; -use uuid::Uuid; type NexusApiDescription = ApiDescription; /// Returns a description of the external nexus API pub(crate) fn external_api() -> NexusApiDescription { - fn register_endpoints( - api: &mut NexusApiDescription, - ) -> Result<(), ApiDescriptionRegisterError> { - api.register(ping)?; - - api.register(system_policy_view)?; - api.register(system_policy_update)?; - - api.register(policy_view)?; - api.register(policy_update)?; - - api.register(project_list)?; - api.register(project_create)?; - api.register(project_view)?; - api.register(project_delete)?; - api.register(project_update)?; - api.register(project_policy_view)?; - api.register(project_policy_update)?; - api.register(project_ip_pool_list)?; - api.register(project_ip_pool_view)?; - - // Operator-Accessible IP Pools API - api.register(ip_pool_list)?; - api.register(ip_pool_create)?; - api.register(ip_pool_silo_list)?; - api.register(ip_pool_silo_link)?; - api.register(ip_pool_silo_unlink)?; - api.register(ip_pool_silo_update)?; - api.register(ip_pool_view)?; - api.register(ip_pool_delete)?; - api.register(ip_pool_update)?; - // Variants for internal services - api.register(ip_pool_service_view)?; - api.register(ip_pool_utilization_view)?; - - // Operator-Accessible IP Pool Range API - api.register(ip_pool_range_list)?; - api.register(ip_pool_range_add)?; - api.register(ip_pool_range_remove)?; - // Variants for internal services - api.register(ip_pool_service_range_list)?; - api.register(ip_pool_service_range_add)?; - api.register(ip_pool_service_range_remove)?; - - api.register(floating_ip_list)?; - api.register(floating_ip_create)?; - api.register(floating_ip_view)?; - api.register(floating_ip_update)?; - api.register(floating_ip_delete)?; - api.register(floating_ip_attach)?; - api.register(floating_ip_detach)?; - - api.register(disk_list)?; - api.register(disk_create)?; - api.register(disk_view)?; - api.register(disk_delete)?; - api.register(disk_metrics_list)?; - - api.register(disk_bulk_write_import_start)?; - api.register(disk_bulk_write_import)?; - api.register(disk_bulk_write_import_stop)?; - api.register(disk_finalize_import)?; - - api.register(instance_list)?; - api.register(instance_view)?; - api.register(instance_create)?; - api.register(instance_delete)?; - api.register(instance_reboot)?; - api.register(instance_start)?; - api.register(instance_stop)?; - api.register(instance_disk_list)?; - api.register(instance_disk_attach)?; - api.register(instance_disk_detach)?; - api.register(instance_serial_console)?; - api.register(instance_serial_console_stream)?; - api.register(instance_ssh_public_key_list)?; - - api.register(image_list)?; - api.register(image_create)?; - api.register(image_view)?; - api.register(image_delete)?; - api.register(image_promote)?; - api.register(image_demote)?; - - api.register(snapshot_list)?; - api.register(snapshot_create)?; - api.register(snapshot_view)?; - api.register(snapshot_delete)?; - - api.register(vpc_list)?; - api.register(vpc_create)?; - api.register(vpc_view)?; - api.register(vpc_update)?; - api.register(vpc_delete)?; - - api.register(vpc_subnet_list)?; - api.register(vpc_subnet_view)?; - api.register(vpc_subnet_create)?; - api.register(vpc_subnet_delete)?; - api.register(vpc_subnet_update)?; - api.register(vpc_subnet_list_network_interfaces)?; - - api.register(instance_network_interface_create)?; - api.register(instance_network_interface_list)?; - api.register(instance_network_interface_view)?; - api.register(instance_network_interface_update)?; - api.register(instance_network_interface_delete)?; - - api.register(instance_external_ip_list)?; - api.register(instance_ephemeral_ip_attach)?; - api.register(instance_ephemeral_ip_detach)?; - - api.register(vpc_router_list)?; - api.register(vpc_router_view)?; - api.register(vpc_router_create)?; - api.register(vpc_router_delete)?; - api.register(vpc_router_update)?; - - api.register(vpc_router_route_list)?; - api.register(vpc_router_route_view)?; - api.register(vpc_router_route_create)?; - api.register(vpc_router_route_delete)?; - api.register(vpc_router_route_update)?; - - api.register(vpc_firewall_rules_view)?; - api.register(vpc_firewall_rules_update)?; - - api.register(rack_list)?; - api.register(rack_view)?; - api.register(sled_list)?; - api.register(sled_view)?; - api.register(sled_set_provision_policy)?; - api.register(sled_instance_list)?; - api.register(sled_physical_disk_list)?; - api.register(physical_disk_list)?; - api.register(physical_disk_view)?; - api.register(switch_list)?; - api.register(switch_view)?; - api.register(sled_list_uninitialized)?; - api.register(sled_add)?; - - api.register(user_builtin_list)?; - api.register(user_builtin_view)?; - - api.register(role_list)?; - api.register(role_view)?; - - api.register(current_user_view)?; - api.register(current_user_groups)?; - api.register(current_user_ssh_key_list)?; - api.register(current_user_ssh_key_view)?; - api.register(current_user_ssh_key_create)?; - api.register(current_user_ssh_key_delete)?; - - // Customer network integration - api.register(networking_address_lot_list)?; - api.register(networking_address_lot_create)?; - api.register(networking_address_lot_delete)?; - api.register(networking_address_lot_block_list)?; - - api.register(networking_loopback_address_create)?; - api.register(networking_loopback_address_delete)?; - api.register(networking_loopback_address_list)?; - - api.register(networking_switch_port_settings_list)?; - api.register(networking_switch_port_settings_view)?; - api.register(networking_switch_port_settings_create)?; - api.register(networking_switch_port_settings_delete)?; - - api.register(networking_switch_port_list)?; - api.register(networking_switch_port_status)?; - api.register(networking_switch_port_apply_settings)?; - api.register(networking_switch_port_clear_settings)?; - - api.register(networking_bgp_config_create)?; - api.register(networking_bgp_config_list)?; - api.register(networking_bgp_status)?; - api.register(networking_bgp_exported)?; - api.register(networking_bgp_imported_routes_ipv4)?; - api.register(networking_bgp_config_delete)?; - api.register(networking_bgp_announce_set_update)?; - api.register(networking_bgp_announce_set_list)?; - api.register(networking_bgp_announce_set_delete)?; - api.register(networking_bgp_message_history)?; - - api.register(networking_bgp_announcement_list)?; - - api.register(networking_bfd_enable)?; - api.register(networking_bfd_disable)?; - api.register(networking_bfd_status)?; - - api.register(networking_allow_list_view)?; - api.register(networking_allow_list_update)?; - - api.register(utilization_view)?; - - // Fleet-wide API operations - api.register(silo_list)?; - api.register(silo_create)?; - api.register(silo_view)?; - api.register(silo_delete)?; - api.register(silo_policy_view)?; - api.register(silo_policy_update)?; - api.register(silo_ip_pool_list)?; - - api.register(silo_utilization_view)?; - api.register(silo_utilization_list)?; - - api.register(system_quotas_list)?; - api.register(silo_quotas_view)?; - api.register(silo_quotas_update)?; - - api.register(silo_identity_provider_list)?; - - api.register(saml_identity_provider_create)?; - api.register(saml_identity_provider_view)?; - - api.register(local_idp_user_create)?; - api.register(local_idp_user_delete)?; - api.register(local_idp_user_set_password)?; - - api.register(certificate_list)?; - api.register(certificate_create)?; - api.register(certificate_view)?; - api.register(certificate_delete)?; - - api.register(system_metric)?; - api.register(silo_metric)?; - api.register(timeseries_schema_list)?; - api.register(timeseries_query)?; - - api.register(system_update_put_repository)?; - api.register(system_update_get_repository)?; - - api.register(user_list)?; - api.register(silo_user_list)?; - api.register(silo_user_view)?; - api.register(group_list)?; - api.register(group_view)?; - - // Console API operations - api.register(console_api::login_begin)?; - api.register(console_api::login_local_begin)?; - api.register(console_api::login_local)?; - api.register(console_api::login_saml_begin)?; - api.register(console_api::login_saml_redirect)?; - api.register(console_api::login_saml)?; - api.register(console_api::logout)?; - - api.register(console_api::console_lookup)?; - api.register(console_api::console_projects)?; - api.register(console_api::console_projects_new)?; - api.register(console_api::console_silo_images)?; - api.register(console_api::console_silo_utilization)?; - api.register(console_api::console_silo_access)?; - api.register(console_api::console_root)?; - api.register(console_api::console_settings_page)?; - api.register(console_api::console_system_page)?; - api.register(console_api::asset)?; - - api.register(device_auth::device_auth_request)?; - api.register(device_auth::device_auth_verify)?; - api.register(device_auth::device_auth_success)?; - api.register(device_auth::device_auth_confirm)?; - api.register(device_auth::device_access_token)?; - - Ok(()) - } - - fn register_experimental( - api: &mut NexusApiDescription, - endpoint: T, - ) -> Result<(), ApiDescriptionRegisterError> - where - T: Into>, + nexus_external_api_mod::api_description::() + .expect("registered entrypoints") +} + +enum NexusExternalApiImpl {} + +impl NexusExternalApi for NexusExternalApiImpl { + type Context = ApiContext; + + async fn system_policy_view( + rqctx: RequestContext, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let policy = nexus.fleet_fetch_policy(&opctx).await?; + Ok(HttpResponseOk(policy)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn system_policy_update( + rqctx: RequestContext, + new_policy: TypedBody>, + ) -> Result>, HttpError> { - let mut ep: ApiEndpoint = endpoint.into(); - // only one tag is allowed - ep.tags = vec![String::from("hidden")]; - ep.path = String::from("/experimental") + &ep.path; - api.register(ep) + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let new_policy = new_policy.into_inner(); + let nasgns = new_policy.role_assignments.len(); + // This should have been validated during parsing. + bail_unless!(nasgns <= shared::MAX_ROLE_ASSIGNMENTS_PER_RESOURCE); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let policy = nexus.fleet_update_policy(&opctx, &new_policy).await?; + Ok(HttpResponseOk(policy)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await } - fn register_experimental_endpoints( - api: &mut NexusApiDescription, - ) -> Result<(), ApiDescriptionRegisterError> { - register_experimental(api, probe_list)?; - register_experimental(api, probe_view)?; - register_experimental(api, probe_create)?; - register_experimental(api, probe_delete)?; + async fn policy_view( + rqctx: RequestContext, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let silo: NameOrId = opctx + .authn + .silo_required() + .internal_context("loading current silo")? + .id() + .into(); + + let silo_lookup = nexus.silo_lookup(&opctx, silo)?; + let policy = nexus.silo_fetch_policy(&opctx, &silo_lookup).await?; + Ok(HttpResponseOk(policy)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - Ok(()) + async fn policy_update( + rqctx: RequestContext, + new_policy: TypedBody>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let new_policy = new_policy.into_inner(); + let nasgns = new_policy.role_assignments.len(); + // This should have been validated during parsing. + bail_unless!(nasgns <= shared::MAX_ROLE_ASSIGNMENTS_PER_RESOURCE); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let silo: NameOrId = opctx + .authn + .silo_required() + .internal_context("loading current silo")? + .id() + .into(); + let silo_lookup = nexus.silo_lookup(&opctx, silo)?; + let policy = nexus + .silo_update_policy(&opctx, &silo_lookup, &new_policy) + .await?; + Ok(HttpResponseOk(policy)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await } - let conf = serde_json::from_str(include_str!("./tag-config.json")).unwrap(); - let mut api = NexusApiDescription::new().tag_config(conf); + async fn utilization_view( + rqctx: RequestContext, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let silo_lookup = nexus.current_silo_lookup(&opctx)?; + let utilization = + nexus.silo_utilization_view(&opctx, &silo_lookup).await?; + + Ok(HttpResponseOk(utilization.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - if let Err(err) = register_endpoints(&mut api) { - panic!("failed to register entrypoints: {}", err); + async fn silo_utilization_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let silo_lookup = + nexus.silo_lookup(&opctx, path_params.into_inner().silo)?; + let quotas = + nexus.silo_utilization_view(&opctx, &silo_lookup).await?; + + Ok(HttpResponseOk(quotas.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await } - if let Err(err) = register_experimental_endpoints(&mut api) { - panic!("failed to register experimental entrypoints: {}", err); + async fn silo_utilization_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let query = query_params.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pagparams, scan_params)?; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let utilization = nexus + .silo_utilization_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + utilization, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await } - api -} -// API ENDPOINT FUNCTION NAMING CONVENTIONS -// -// Generally, HTTP resources are grouped within some collection. For a -// relatively simple example: -// -// GET v1/projects (list the projects in the collection) -// POST v1/projects (create a project in the collection) -// GET v1/projects/{project} (look up a project in the collection) -// DELETE v1/projects/{project} (delete a project in the collection) -// PUT v1/projects/{project} (update a project in the collection) -// -// We pick a name for the function that implements a given API entrypoint -// based on how we expect it to appear in the CLI subcommand hierarchy. For -// example: -// -// GET v1/projects -> project_list() -// POST v1/projects -> project_create() -// GET v1/projects/{project} -> project_view() -// DELETE v1/projects/{project} -> project_delete() -// PUT v1/projects/{project} -> project_update() -// -// Note that the path typically uses the entity's plural form while the -// function name uses its singular. -// -// Operations beyond list, create, view, delete, and update should use a -// descriptive noun or verb, again bearing in mind that this will be -// transcribed into the CLI and SDKs: -// -// POST -> instance_reboot -// POST -> instance_stop -// GET -> instance_serial_console -// -// Note that these function names end up in generated OpenAPI spec as the -// operationId for each endpoint, and therefore represent a contract with -// clients. Client generators use operationId to name API methods, so changing -// a function name is a breaking change from a client perspective. - -/// Ping API -/// -/// Always responds with Ok if it responds at all. -#[endpoint { - method = GET, - path = "/v1/ping", - tags = ["system/status"], -}] -async fn ping( - _rqctx: RequestContext, -) -> Result, HttpError> { - Ok(HttpResponseOk(views::Ping { status: views::PingStatus::Ok })) -} + async fn system_quotas_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let query = query_params.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)?; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let quotas = nexus + .fleet_list_quotas(&opctx, &pagparams) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanById::results_page( + &query, + quotas, + &|_, quota: &SiloQuotas| quota.silo_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch top-level IAM policy -#[endpoint { - method = GET, - path = "/v1/system/policy", - tags = ["policy"], -}] -async fn system_policy_view( - rqctx: RequestContext, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let policy = nexus.fleet_fetch_policy(&opctx).await?; - Ok(HttpResponseOk(policy)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn silo_quotas_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let silo_lookup = + nexus.silo_lookup(&opctx, path_params.into_inner().silo)?; + let quota = nexus.silo_quotas_view(&opctx, &silo_lookup).await?; + Ok(HttpResponseOk(quota.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update top-level IAM policy -#[endpoint { - method = PUT, - path = "/v1/system/policy", - tags = ["policy"], -}] -async fn system_policy_update( - rqctx: RequestContext, - new_policy: TypedBody>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let new_policy = new_policy.into_inner(); - let nasgns = new_policy.role_assignments.len(); - // This should have been validated during parsing. - bail_unless!(nasgns <= shared::MAX_ROLE_ASSIGNMENTS_PER_RESOURCE); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let policy = nexus.fleet_update_policy(&opctx, &new_policy).await?; - Ok(HttpResponseOk(policy)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn silo_quotas_update( + rqctx: RequestContext, + path_params: Path, + new_quota: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let silo_lookup = + nexus.silo_lookup(&opctx, path_params.into_inner().silo)?; + let quota = nexus + .silo_update_quota( + &opctx, + &silo_lookup, + &new_quota.into_inner(), + ) + .await?; + Ok(HttpResponseOk(quota.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch current silo's IAM policy -#[endpoint { - method = GET, - path = "/v1/policy", - tags = ["silos"], - }] -pub(crate) async fn policy_view( - rqctx: RequestContext, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let silo: NameOrId = opctx - .authn - .silo_required() - .internal_context("loading current silo")? - .id() - .into(); - - let silo_lookup = nexus.silo_lookup(&opctx, silo)?; - let policy = nexus.silo_fetch_policy(&opctx, &silo_lookup).await?; - Ok(HttpResponseOk(policy)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn silo_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let silos = nexus + .silos_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.try_into()) + .collect::, Error>>()?; + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + silos, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update current silo's IAM policy -#[endpoint { - method = PUT, - path = "/v1/policy", - tags = ["silos"], -}] -async fn policy_update( - rqctx: RequestContext, - new_policy: TypedBody>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let new_policy = new_policy.into_inner(); - let nasgns = new_policy.role_assignments.len(); - // This should have been validated during parsing. - bail_unless!(nasgns <= shared::MAX_ROLE_ASSIGNMENTS_PER_RESOURCE); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let silo: NameOrId = opctx - .authn - .silo_required() - .internal_context("loading current silo")? - .id() - .into(); - let silo_lookup = nexus.silo_lookup(&opctx, silo)?; - let policy = - nexus.silo_update_policy(&opctx, &silo_lookup, &new_policy).await?; - Ok(HttpResponseOk(policy)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn silo_create( + rqctx: RequestContext, + new_silo_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let silo = + nexus.silo_create(&opctx, new_silo_params.into_inner()).await?; + Ok(HttpResponseCreated(silo.try_into()?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch resource utilization for user's current silo -#[endpoint { - method = GET, - path = "/v1/utilization", - tags = ["silos"], -}] -async fn utilization_view( - rqctx: RequestContext, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let silo_lookup = nexus.current_silo_lookup(&opctx)?; - let utilization = - nexus.silo_utilization_view(&opctx, &silo_lookup).await?; - - Ok(HttpResponseOk(utilization.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn silo_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; + let (.., silo) = silo_lookup.fetch().await?; + Ok(HttpResponseOk(silo.try_into()?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch current utilization for given silo -#[endpoint { - method = GET, - path = "/v1/system/utilization/silos/{silo}", - tags = ["system/silos"], -}] -async fn silo_utilization_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; + async fn silo_ip_pool_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + + let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; + let pools = nexus + .silo_ip_pool_list(&opctx, &silo_lookup, &paginated_by) + .await? + .iter() + .map(|(pool, silo_link)| views::SiloIpPool { + identity: pool.identity(), + is_default: silo_link.is_default, + }) + .collect(); + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + pools, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let silo_lookup = - nexus.silo_lookup(&opctx, path_params.into_inner().silo)?; - let quotas = nexus.silo_utilization_view(&opctx, &silo_lookup).await?; - - Ok(HttpResponseOk(quotas.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} -/// List current utilization state for all silos -#[endpoint { - method = GET, - path = "/v1/system/utilization/silos", - tags = ["system/silos"], -}] -async fn silo_utilization_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; + async fn silo_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let params = path_params.into_inner(); + let silo_lookup = nexus.silo_lookup(&opctx, params.silo)?; + nexus.silo_delete(&opctx, &silo_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let query = query_params.into_inner(); - let pagparams = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pagparams, scan_params)?; + async fn silo_policy_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; + let policy = nexus.silo_fetch_policy(&opctx, &silo_lookup).await?; + Ok(HttpResponseOk(policy)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let utilization = nexus - .silo_utilization_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - utilization, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn silo_policy_update( + rqctx: RequestContext, + path_params: Path, + new_policy: TypedBody>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let new_policy = new_policy.into_inner(); + let nasgns = new_policy.role_assignments.len(); + // This should have been validated during parsing. + bail_unless!(nasgns <= shared::MAX_ROLE_ASSIGNMENTS_PER_RESOURCE); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; + let policy = nexus + .silo_update_policy(&opctx, &silo_lookup, &new_policy) + .await?; + Ok(HttpResponseOk(policy)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Lists resource quotas for all silos -#[endpoint { - method = GET, - path = "/v1/system/silo-quotas", - tags = ["system/silos"], -}] -async fn system_quotas_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; + // Silo-specific user endpoints + + async fn silo_user_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanById::from_query(&query)?; + let silo_lookup = + nexus.silo_lookup(&opctx, scan_params.selector.silo.clone())?; + let users = nexus + .silo_list_users(&opctx, &silo_lookup, &pag_params) + .await? + .into_iter() + .map(|i| i.into()) + .collect(); + Ok(HttpResponseOk(ScanById::results_page( + &query, + users, + &|_, user: &User| user.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let query = query_params.into_inner(); - let pagparams = data_page_params_for(&rqctx, &query)?; + async fn silo_user_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; + let user = nexus + .silo_user_fetch(&opctx, &silo_lookup, path.user_id) + .await?; + Ok(HttpResponseOk(user.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let quotas = nexus - .fleet_list_quotas(&opctx, &pagparams) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(ScanById::results_page( - &query, - quotas, - &|_, quota: &SiloQuotas| quota.silo_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Silo identity providers + + async fn silo_identity_provider_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let silo_lookup = + nexus.silo_lookup(&opctx, scan_params.selector.silo.clone())?; + let identity_providers = nexus + .identity_provider_list(&opctx, &silo_lookup, &paginated_by) + .await? + .into_iter() + .map(|x| x.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + identity_providers, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch resource quotas for silo -#[endpoint { - method = GET, - path = "/v1/system/silos/{silo}/quotas", - tags = ["system/silos"], -}] -async fn silo_quotas_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; + // Silo SAML identity providers - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let silo_lookup = - nexus.silo_lookup(&opctx, path_params.into_inner().silo)?; - let quota = nexus.silo_quotas_view(&opctx, &silo_lookup).await?; - Ok(HttpResponseOk(quota.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn saml_identity_provider_create( + rqctx: RequestContext, + query_params: Query, + new_provider: TypedBody, + ) -> Result, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; + let provider = nexus + .saml_identity_provider_create( + &opctx, + &silo_lookup, + new_provider.into_inner(), + ) + .await?; + Ok(HttpResponseCreated(provider.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update resource quotas for silo -/// -/// If a quota value is not specified, it will remain unchanged. -#[endpoint { - method = PUT, - path = "/v1/system/silos/{silo}/quotas", - tags = ["system/silos"], -}] -async fn silo_quotas_update( - rqctx: RequestContext, - path_params: Path, - new_quota: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; + async fn saml_identity_provider_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let saml_identity_provider_selector = + params::SamlIdentityProviderSelector { + silo: Some(query.silo), + saml_identity_provider: path.provider, + }; + let (.., provider) = nexus + .saml_identity_provider_lookup( + &opctx, + saml_identity_provider_selector, + )? + .fetch() + .await?; + Ok(HttpResponseOk(provider.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let silo_lookup = - nexus.silo_lookup(&opctx, path_params.into_inner().silo)?; - let quota = nexus - .silo_update_quota(&opctx, &silo_lookup, &new_quota.into_inner()) - .await?; - Ok(HttpResponseOk(quota.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // TODO: no DELETE for identity providers? + + // "Local" Identity Provider + + async fn local_idp_user_create( + rqctx: RequestContext, + query_params: Query, + new_user_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; + let user = nexus + .local_idp_create_user( + &opctx, + &silo_lookup, + new_user_params.into_inner(), + ) + .await?; + Ok(HttpResponseCreated(user.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List silos -/// -/// Lists silos that are discoverable based on the current permissions. -#[endpoint { - method = GET, - path = "/v1/system/silos", - tags = ["system/silos"], -}] -async fn silo_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let silos = nexus - .silos_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(|p| p.try_into()) - .collect::, Error>>()?; - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - silos, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn local_idp_user_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; + nexus + .local_idp_delete_user(&opctx, &silo_lookup, path.user_id) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create a silo -#[endpoint { - method = POST, - path = "/v1/system/silos", - tags = ["system/silos"], -}] -async fn silo_create( - rqctx: RequestContext, - new_silo_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let silo = - nexus.silo_create(&opctx, new_silo_params.into_inner()).await?; - Ok(HttpResponseCreated(silo.try_into()?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn local_idp_user_set_password( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + update: TypedBody, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; + nexus + .local_idp_user_set_password( + &opctx, + &silo_lookup, + path.user_id, + update.into_inner(), + ) + .await?; + Ok(HttpResponseUpdatedNoContent()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch silo -/// -/// Fetch silo by name or ID. -#[endpoint { - method = GET, - path = "/v1/system/silos/{silo}", - tags = ["system/silos"], -}] -async fn silo_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn project_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let projects = nexus + .project_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + projects, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn project_create( + rqctx: RequestContext, + new_project: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let nexus = &apictx.context.nexus; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project = + nexus.project_create(&opctx, &new_project.into_inner()).await?; + Ok(HttpResponseCreated(project.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn project_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); - let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; - let (.., silo) = silo_lookup.fetch().await?; - Ok(HttpResponseOk(silo.try_into()?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_selector = + params::ProjectSelector { project: path.project }; + let (.., project) = + nexus.project_lookup(&opctx, project_selector)?.fetch().await?; + Ok(HttpResponseOk(project.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List IP pools linked to silo -/// -/// Linked IP pools are available to users in the specified silo. A silo can -/// have at most one default pool. IPs are allocated from the default pool when -/// users ask for one without specifying a pool. -#[endpoint { - method = GET, - path = "/v1/system/silos/{silo}/ip-pools", - tags = ["system/silos"], -}] -async fn silo_ip_pool_list( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn project_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_selector = + params::ProjectSelector { project: path.project }; + let project_lookup = + nexus.project_lookup(&opctx, project_selector)?; + nexus.project_delete(&opctx, &project_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - - let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; - let pools = nexus - .silo_ip_pool_list(&opctx, &silo_lookup, &paginated_by) - .await? - .iter() - .map(|(pool, silo_link)| views::SiloIpPool { - identity: pool.identity(), - is_default: silo_link.is_default, - }) - .collect(); - - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - pools, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Delete a silo -/// -/// Delete a silo by name or ID. -#[endpoint { - method = DELETE, - path = "/v1/system/silos/{silo}", - tags = ["system/silos"], -}] -async fn silo_delete( - rqctx: RequestContext, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + // TODO-correctness: Is it valid for PUT to accept application/json that's a + // subset of what the resource actually represents? If not, is that a problem? + // (HTTP may require that this be idempotent.) If so, can we get around that + // having this be a slightly different content-type (e.g., + // "application/json-patch")? We should see what other APIs do. + async fn project_update( + rqctx: RequestContext, + path_params: Path, + updated_project: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; - let params = path_params.into_inner(); - let silo_lookup = nexus.silo_lookup(&opctx, params.silo)?; - nexus.silo_delete(&opctx, &silo_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let path = path_params.into_inner(); + let updated_project = updated_project.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_selector = + params::ProjectSelector { project: path.project }; + let project_lookup = + nexus.project_lookup(&opctx, project_selector)?; + let project = nexus + .project_update(&opctx, &project_lookup, &updated_project) + .await?; + Ok(HttpResponseOk(project.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch silo IAM policy -#[endpoint { - method = GET, - path = "/v1/system/silos/{silo}/policy", - tags = ["system/silos"], -}] -async fn silo_policy_view( - rqctx: RequestContext, - path_params: Path, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn project_policy_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); - let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; - let policy = nexus.silo_fetch_policy(&opctx, &silo_lookup).await?; - Ok(HttpResponseOk(policy)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_selector = + params::ProjectSelector { project: path.project }; + let project_lookup = + nexus.project_lookup(&opctx, project_selector)?; + let policy = + nexus.project_fetch_policy(&opctx, &project_lookup).await?; + Ok(HttpResponseOk(policy)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update silo IAM policy -#[endpoint { - method = PUT, - path = "/v1/system/silos/{silo}/policy", - tags = ["system/silos"], -}] -async fn silo_policy_update( - rqctx: RequestContext, - path_params: Path, - new_policy: TypedBody>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let new_policy = new_policy.into_inner(); - let nasgns = new_policy.role_assignments.len(); - // This should have been validated during parsing. - bail_unless!(nasgns <= shared::MAX_ROLE_ASSIGNMENTS_PER_RESOURCE); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn project_policy_update( + rqctx: RequestContext, + path_params: Path, + new_policy: TypedBody>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); - let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; - let policy = - nexus.silo_update_policy(&opctx, &silo_lookup, &new_policy).await?; - Ok(HttpResponseOk(policy)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let new_policy = new_policy.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_selector = + params::ProjectSelector { project: path.project }; + let project_lookup = + nexus.project_lookup(&opctx, project_selector)?; + nexus + .project_update_policy(&opctx, &project_lookup, &new_policy) + .await?; + Ok(HttpResponseOk(new_policy)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Silo-specific user endpoints - -/// List built-in (system) users in silo -#[endpoint { - method = GET, - path = "/v1/system/users", - tags = ["system/silos"], -}] -async fn silo_user_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanById::from_query(&query)?; - let silo_lookup = - nexus.silo_lookup(&opctx, scan_params.selector.silo.clone())?; - let users = nexus - .silo_list_users(&opctx, &silo_lookup, &pag_params) - .await? - .into_iter() - .map(|i| i.into()) - .collect(); - Ok(HttpResponseOk(ScanById::results_page( - &query, - users, - &|_, user: &User| user.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // IP Pools + + async fn project_ip_pool_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let pools = nexus + .current_silo_ip_pool_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|(pool, silo_link)| views::SiloIpPool { + identity: pool.identity(), + is_default: silo_link.is_default, + }) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + pools, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Path parameters for Silo User requests -#[derive(Deserialize, JsonSchema)] -struct UserParam { - /// The user's internal id - user_id: Uuid, -} + async fn project_ip_pool_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let pool_selector = path_params.into_inner().pool; + let (pool, silo_link) = + nexus.silo_ip_pool_fetch(&opctx, &pool_selector).await?; + Ok(HttpResponseOk(views::SiloIpPool { + identity: pool.identity(), + is_default: silo_link.is_default, + })) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch built-in (system) user -#[endpoint { - method = GET, - path = "/v1/system/users/{user_id}", - tags = ["system/silos"], -}] -async fn silo_user_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; - let user = - nexus.silo_user_fetch(&opctx, &silo_lookup, path.user_id).await?; - Ok(HttpResponseOk(user.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let pools = nexus + .ip_pools_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(IpPool::from) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + pools, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Silo identity providers - -/// List a silo's IdP's name -#[endpoint { - method = GET, - path = "/v1/system/identity-providers", - tags = ["system/silos"], -}] -async fn silo_identity_provider_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let silo_lookup = - nexus.silo_lookup(&opctx, scan_params.selector.silo.clone())?; - let identity_providers = nexus - .identity_provider_list(&opctx, &silo_lookup, &paginated_by) - .await? - .into_iter() - .map(|x| x.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - identity_providers, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_create( + rqctx: RequestContext, + pool_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let nexus = &apictx.context.nexus; + let pool_params = pool_params.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let pool = nexus.ip_pool_create(&opctx, &pool_params).await?; + Ok(HttpResponseCreated(IpPool::from(pool))) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Silo SAML identity providers - -/// Create SAML IdP -#[endpoint { - method = POST, - path = "/v1/system/identity-providers/saml", - tags = ["system/silos"], -}] -async fn saml_identity_provider_create( - rqctx: RequestContext, - query_params: Query, - new_provider: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; - let provider = nexus - .saml_identity_provider_create( - &opctx, - &silo_lookup, - new_provider.into_inner(), - ) - .await?; - Ok(HttpResponseCreated(provider.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let pool_selector = path_params.into_inner().pool; + // We do not prevent the service pool from being fetched by name or ID + // like we do for update, delete, associate. + let (.., pool) = + nexus.ip_pool_lookup(&opctx, &pool_selector)?.fetch().await?; + Ok(HttpResponseOk(IpPool::from(pool))) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch SAML IdP -#[endpoint { - method = GET, - path = "/v1/system/identity-providers/saml/{provider}", - tags = ["system/silos"], -}] -async fn saml_identity_provider_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let saml_identity_provider_selector = - params::SamlIdentityProviderSelector { - silo: Some(query.silo), - saml_identity_provider: path.provider, - }; - let (.., provider) = nexus - .saml_identity_provider_lookup( - &opctx, - saml_identity_provider_selector, - )? - .fetch() - .await?; - Ok(HttpResponseOk(provider.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; + nexus.ip_pool_delete(&opctx, &pool_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// TODO: no DELETE for identity providers? - -// "Local" Identity Provider - -/// Create user -/// -/// Users can only be created in Silos with `provision_type` == `Fixed`. -/// Otherwise, Silo users are just-in-time (JIT) provisioned when a user first -/// logs in using an external Identity Provider. -#[endpoint { - method = POST, - path = "/v1/system/identity-providers/local/users", - tags = ["system/silos"], -}] -async fn local_idp_user_create( - rqctx: RequestContext, - query_params: Query, - new_user_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; - let user = nexus - .local_idp_create_user( - &opctx, - &silo_lookup, - new_user_params.into_inner(), - ) - .await?; - Ok(HttpResponseCreated(user.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Delete user -#[endpoint { - method = DELETE, - path = "/v1/system/identity-providers/local/users/{user_id}", - tags = ["system/silos"], -}] -async fn local_idp_user_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; - nexus.local_idp_delete_user(&opctx, &silo_lookup, path.user_id).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_update( + rqctx: RequestContext, + path_params: Path, + updates: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let updates = updates.into_inner(); + let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; + let pool = + nexus.ip_pool_update(&opctx, &pool_lookup, &updates).await?; + Ok(HttpResponseOk(pool.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Set or invalidate user's password -/// -/// Passwords can only be updated for users in Silos with identity mode -/// `LocalOnly`. -#[endpoint { - method = POST, - path = "/v1/system/identity-providers/local/users/{user_id}/set-password", - tags = ["system/silos"], -}] -async fn local_idp_user_set_password( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - update: TypedBody, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let silo_lookup = nexus.silo_lookup(&opctx, query.silo)?; - nexus - .local_idp_user_set_password( - &opctx, - &silo_lookup, - path.user_id, - update.into_inner(), - ) - .await?; - Ok(HttpResponseUpdatedNoContent()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_utilization_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let pool_selector = path_params.into_inner().pool; + // We do not prevent the service pool from being fetched by name or ID + // like we do for update, delete, associate. + let pool_lookup = nexus.ip_pool_lookup(&opctx, &pool_selector)?; + let utilization = + nexus.ip_pool_utilization_view(&opctx, &pool_lookup).await?; + Ok(HttpResponseOk(utilization.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List projects -#[endpoint { - method = GET, - path = "/v1/projects", - tags = ["projects"], -}] -async fn project_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let projects = nexus - .project_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - projects, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_silo_list( + rqctx: RequestContext, + path_params: Path, + // paginating by resource_id because they're unique per pool. most robust + // option would be to paginate by a composite key representing the (pool, + // resource_type, resource) + query_params: Query, + // TODO: this could just list views::Silo -- it's not like knowing silo_id + // and nothing else is particularly useful -- except we also want to say + // whether the pool is marked default on each silo. So one option would + // be to do the same as we did with SiloIpPool -- include is_default on + // whatever the thing is. Still... all we'd have to do to make this usable + // in both places would be to make it { ...IpPool, silo_id, silo_name, + // is_default } + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; -/// Create project -#[endpoint { - method = POST, - path = "/v1/projects", - tags = ["projects"], -}] -async fn project_create( - rqctx: RequestContext, - new_project: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project = - nexus.project_create(&opctx, &new_project.into_inner()).await?; - Ok(HttpResponseCreated(project.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; -/// Fetch project -#[endpoint { - method = GET, - path = "/v1/projects/{project}", - tags = ["projects"], -}] -async fn project_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_selector = - params::ProjectSelector { project: path.project }; - let (.., project) = - nexus.project_lookup(&opctx, project_selector)?.fetch().await?; - Ok(HttpResponseOk(project.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let path = path_params.into_inner(); + let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; -/// Delete project -#[endpoint { - method = DELETE, - path = "/v1/projects/{project}", - tags = ["projects"], -}] -async fn project_delete( - rqctx: RequestContext, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_selector = - params::ProjectSelector { project: path.project }; - let project_lookup = nexus.project_lookup(&opctx, project_selector)?; - nexus.project_delete(&opctx, &project_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let assocs = nexus + .ip_pool_silo_list(&opctx, &pool_lookup, &pag_params) + .await? + .into_iter() + .map(|assoc| assoc.into()) + .collect(); + + Ok(HttpResponseOk(ScanById::results_page( + &query, + assocs, + &|_, x: &views::IpPoolSiloLink| x.silo_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// TODO-correctness: Is it valid for PUT to accept application/json that's a -// subset of what the resource actually represents? If not, is that a problem? -// (HTTP may require that this be idempotent.) If so, can we get around that -// having this be a slightly different content-type (e.g., -// "application/json-patch")? We should see what other APIs do. -/// Update a project -#[endpoint { - method = PUT, - path = "/v1/projects/{project}", - tags = ["projects"], -}] -async fn project_update( - rqctx: RequestContext, - path_params: Path, - updated_project: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let updated_project = updated_project.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_selector = - params::ProjectSelector { project: path.project }; - let project_lookup = nexus.project_lookup(&opctx, project_selector)?; - let project = nexus - .project_update(&opctx, &project_lookup, &updated_project) - .await?; - Ok(HttpResponseOk(project.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_silo_link( + rqctx: RequestContext, + path_params: Path, + resource_assoc: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let resource_assoc = resource_assoc.into_inner(); + let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; + let assoc = nexus + .ip_pool_link_silo(&opctx, &pool_lookup, &resource_assoc) + .await?; + Ok(HttpResponseCreated(assoc.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch project's IAM policy -#[endpoint { - method = GET, - path = "/v1/projects/{project}/policy", - tags = ["projects"], -}] -async fn project_policy_view( - rqctx: RequestContext, - path_params: Path, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_selector = - params::ProjectSelector { project: path.project }; - let project_lookup = nexus.project_lookup(&opctx, project_selector)?; - let policy = - nexus.project_fetch_policy(&opctx, &project_lookup).await?; - Ok(HttpResponseOk(policy)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_silo_unlink( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; + let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; + nexus + .ip_pool_unlink_silo(&opctx, &pool_lookup, &silo_lookup) + .await?; + Ok(HttpResponseUpdatedNoContent()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update project's IAM policy -#[endpoint { - method = PUT, - path = "/v1/projects/{project}/policy", - tags = ["projects"], -}] -async fn project_policy_update( - rqctx: RequestContext, - path_params: Path, - new_policy: TypedBody>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let new_policy = new_policy.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_selector = - params::ProjectSelector { project: path.project }; - let project_lookup = nexus.project_lookup(&opctx, project_selector)?; - nexus - .project_update_policy(&opctx, &project_lookup, &new_policy) - .await?; - Ok(HttpResponseOk(new_policy)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_silo_update( + rqctx: RequestContext, + path_params: Path, + update: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let update = update.into_inner(); + let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; + let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; + let assoc = nexus + .ip_pool_silo_update( + &opctx, + &pool_lookup, + &silo_lookup, + &update, + ) + .await?; + Ok(HttpResponseOk(assoc.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// IP Pools - -/// List IP pools -#[endpoint { - method = GET, - path = "/v1/ip-pools", - tags = ["projects"], -}] -async fn project_ip_pool_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { + async fn ip_pool_service_view( + rqctx: RequestContext, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let pools = nexus - .current_silo_ip_pool_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(|(pool, silo_link)| views::SiloIpPool { - identity: pool.identity(), - is_default: silo_link.is_default, - }) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - pools, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let pool = nexus.ip_pool_service_fetch(&opctx).await?; + Ok(HttpResponseOk(IpPool::from(pool))) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch IP pool -#[endpoint { - method = GET, - path = "/v1/ip-pools/{pool}", - tags = ["projects"], -}] -async fn project_ip_pool_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let pool_selector = path_params.into_inner().pool; - let (pool, silo_link) = - nexus.silo_ip_pool_fetch(&opctx, &pool_selector).await?; - Ok(HttpResponseOk(views::SiloIpPool { - identity: pool.identity(), - is_default: silo_link.is_default, - })) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_range_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let path = path_params.into_inner(); + let marker = match query.page { + WhichPage::First(_) => None, + WhichPage::Next(ref addr) => Some(addr), + }; + let pag_params = DataPageParams { + limit: rqctx.page_limit(&query)?, + direction: PaginationOrder::Ascending, + marker, + }; + let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; + let ranges = nexus + .ip_pool_list_ranges(&opctx, &pool_lookup, &pag_params) + .await? + .into_iter() + .map(|range| range.into()) + .collect(); + Ok(HttpResponseOk(ResultsPage::new( + ranges, + &EmptyScanParams {}, + |range: &IpPoolRange, _| { + IpNetwork::from(range.range.first_address()) + }, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List IP pools -#[endpoint { - method = GET, - path = "/v1/system/ip-pools", - tags = ["system/networking"], -}] -async fn ip_pool_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let pools = nexus - .ip_pools_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(IpPool::from) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - pools, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_range_add( + rqctx: RequestContext, + path_params: Path, + range_params: TypedBody, + ) -> Result, HttpError> { + let apictx = &rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let range = range_params.into_inner(); + let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; + let out = + nexus.ip_pool_add_range(&opctx, &pool_lookup, &range).await?; + Ok(HttpResponseCreated(out.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create IP pool -#[endpoint { - method = POST, - path = "/v1/system/ip-pools", - tags = ["system/networking"], -}] -async fn ip_pool_create( - rqctx: RequestContext, - pool_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let pool_params = pool_params.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let pool = nexus.ip_pool_create(&opctx, &pool_params).await?; - Ok(HttpResponseCreated(IpPool::from(pool))) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_range_remove( + rqctx: RequestContext, + path_params: Path, + range_params: TypedBody, + ) -> Result { + let apictx = &rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let range = range_params.into_inner(); + let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; + nexus.ip_pool_delete_range(&opctx, &pool_lookup, &range).await?; + Ok(HttpResponseUpdatedNoContent()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch IP pool -#[endpoint { - method = GET, - path = "/v1/system/ip-pools/{pool}", - tags = ["system/networking"], -}] -async fn ip_pool_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let pool_selector = path_params.into_inner().pool; - // We do not prevent the service pool from being fetched by name or ID - // like we do for update, delete, associate. - let (.., pool) = - nexus.ip_pool_lookup(&opctx, &pool_selector)?.fetch().await?; - Ok(HttpResponseOk(IpPool::from(pool))) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_service_range_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let marker = match query.page { + WhichPage::First(_) => None, + WhichPage::Next(ref addr) => Some(addr), + }; + let pag_params = DataPageParams { + limit: rqctx.page_limit(&query)?, + direction: PaginationOrder::Ascending, + marker, + }; + let ranges = nexus + .ip_pool_service_list_ranges(&opctx, &pag_params) + .await? + .into_iter() + .map(|range| range.into()) + .collect(); + Ok(HttpResponseOk(ResultsPage::new( + ranges, + &EmptyScanParams {}, + |range: &IpPoolRange, _| { + IpNetwork::from(range.range.first_address()) + }, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete IP pool -#[endpoint { - method = DELETE, - path = "/v1/system/ip-pools/{pool}", - tags = ["system/networking"], -}] -async fn ip_pool_delete( - rqctx: RequestContext, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; - nexus.ip_pool_delete(&opctx, &pool_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn ip_pool_service_range_add( + rqctx: RequestContext, + range_params: TypedBody, + ) -> Result, HttpError> { + let apictx = &rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let range = range_params.into_inner(); + let out = nexus.ip_pool_service_add_range(&opctx, &range).await?; + Ok(HttpResponseCreated(out.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update IP pool -#[endpoint { - method = PUT, - path = "/v1/system/ip-pools/{pool}", - tags = ["system/networking"], -}] -async fn ip_pool_update( - rqctx: RequestContext, - path_params: Path, - updates: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn ip_pool_service_range_remove( + rqctx: RequestContext, + range_params: TypedBody, + ) -> Result { + let apictx = &rqctx.context(); let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let updates = updates.into_inner(); - let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; - let pool = nexus.ip_pool_update(&opctx, &pool_lookup, &updates).await?; - Ok(HttpResponseOk(pool.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let range = range_params.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + nexus.ip_pool_service_delete_range(&opctx, &range).await?; + Ok(HttpResponseUpdatedNoContent()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch IP pool utilization -#[endpoint { - method = GET, - path = "/v1/system/ip-pools/{pool}/utilization", - tags = ["system/networking"], -}] -async fn ip_pool_utilization_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let pool_selector = path_params.into_inner().pool; - // We do not prevent the service pool from being fetched by name or ID - // like we do for update, delete, associate. - let pool_lookup = nexus.ip_pool_lookup(&opctx, &pool_selector)?; - let utilization = - nexus.ip_pool_utilization_view(&opctx, &pool_lookup).await?; - Ok(HttpResponseOk(utilization.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Floating IP Addresses + + async fn floating_ip_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let ips = nexus + .floating_ips_list(&opctx, &project_lookup, &paginated_by) + .await?; + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + ips, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List IP pool's linked silos -#[endpoint { - method = GET, - path = "/v1/system/ip-pools/{pool}/silos", - tags = ["system/networking"], -}] -async fn ip_pool_silo_list( - rqctx: RequestContext, - path_params: Path, - // paginating by resource_id because they're unique per pool. most robust - // option would be to paginate by a composite key representing the (pool, - // resource_type, resource) - query_params: Query, - // TODO: this could just list views::Silo -- it's not like knowing silo_id - // and nothing else is particularly useful -- except we also want to say - // whether the pool is marked default on each silo. So one option would - // be to do the same as we did with SiloIpPool -- include is_default on - // whatever the thing is. Still... all we'd have to do to make this usable - // in both places would be to make it { ...IpPool, silo_id, silo_name, - // is_default } -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; + async fn floating_ip_create( + rqctx: RequestContext, + query_params: Query, + floating_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let nexus = &apictx.context.nexus; + let floating_params = floating_params.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_lookup = + nexus.project_lookup(&opctx, query_params.into_inner())?; + let ip = nexus + .floating_ip_create(&opctx, &project_lookup, floating_params) + .await?; + Ok(HttpResponseCreated(ip)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; + async fn floating_ip_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + updated_floating_ip: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let updated_floating_ip_params = updated_floating_ip.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let floating_ip_selector = params::FloatingIpSelector { + project: query.project, + floating_ip: path.floating_ip, + }; + let floating_ip_lookup = + nexus.floating_ip_lookup(&opctx, floating_ip_selector)?; + let floating_ip = nexus + .floating_ip_update( + &opctx, + floating_ip_lookup, + updated_floating_ip_params, + ) + .await?; + Ok(HttpResponseOk(floating_ip)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let path = path_params.into_inner(); - let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; - - let assocs = nexus - .ip_pool_silo_list(&opctx, &pool_lookup, &pag_params) - .await? - .into_iter() - .map(|assoc| assoc.into()) - .collect(); - - Ok(HttpResponseOk(ScanById::results_page( - &query, - assocs, - &|_, x: &views::IpPoolSiloLink| x.silo_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn floating_ip_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let floating_ip_selector = params::FloatingIpSelector { + floating_ip: path.floating_ip, + project: query.project, + }; + let fip_lookup = + nexus.floating_ip_lookup(&opctx, floating_ip_selector)?; -/// Link IP pool to silo -/// -/// Users in linked silos can allocate external IPs from this pool for their -/// instances. A silo can have at most one default pool. IPs are allocated from -/// the default pool when users ask for one without specifying a pool. -#[endpoint { - method = POST, - path = "/v1/system/ip-pools/{pool}/silos", - tags = ["system/networking"], -}] -async fn ip_pool_silo_link( - rqctx: RequestContext, - path_params: Path, - resource_assoc: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let resource_assoc = resource_assoc.into_inner(); - let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; - let assoc = nexus - .ip_pool_link_silo(&opctx, &pool_lookup, &resource_assoc) - .await?; - Ok(HttpResponseCreated(assoc.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + nexus.floating_ip_delete(&opctx, fip_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Unlink IP pool from silo -/// -/// Will fail if there are any outstanding IPs allocated in the silo. -#[endpoint { - method = DELETE, - path = "/v1/system/ip-pools/{pool}/silos/{silo}", - tags = ["system/networking"], -}] -async fn ip_pool_silo_unlink( - rqctx: RequestContext, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; - let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; - nexus.ip_pool_unlink_silo(&opctx, &pool_lookup, &silo_lookup).await?; - Ok(HttpResponseUpdatedNoContent()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn floating_ip_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let floating_ip_selector = params::FloatingIpSelector { + floating_ip: path.floating_ip, + project: query.project, + }; + let (.., fip) = nexus + .floating_ip_lookup(&opctx, floating_ip_selector)? + .fetch() + .await?; + Ok(HttpResponseOk(fip.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Make IP pool default for silo -/// -/// When a user asks for an IP (e.g., at instance create time) without -/// specifying a pool, the IP comes from the default pool if a default is -/// configured. When a pool is made the default for a silo, any existing default -/// will remain linked to the silo, but will no longer be the default. -#[endpoint { - method = PUT, - path = "/v1/system/ip-pools/{pool}/silos/{silo}", - tags = ["system/networking"], -}] -async fn ip_pool_silo_update( - rqctx: RequestContext, - path_params: Path, - update: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let update = update.into_inner(); - let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; - let silo_lookup = nexus.silo_lookup(&opctx, path.silo)?; - let assoc = nexus - .ip_pool_silo_update(&opctx, &pool_lookup, &silo_lookup, &update) - .await?; - Ok(HttpResponseOk(assoc.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn floating_ip_attach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + target: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let floating_ip_selector = params::FloatingIpSelector { + floating_ip: path.floating_ip, + project: query.project, + }; + let ip = nexus + .floating_ip_attach( + &opctx, + floating_ip_selector, + target.into_inner(), + ) + .await?; + Ok(HttpResponseAccepted(ip)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch Oxide service IP pool -#[endpoint { - method = GET, - path = "/v1/system/ip-pools-service", - tags = ["system/networking"], -}] -async fn ip_pool_service_view( - rqctx: RequestContext, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let pool = nexus.ip_pool_service_fetch(&opctx).await?; - Ok(HttpResponseOk(IpPool::from(pool))) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn floating_ip_detach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let floating_ip_selector = params::FloatingIpSelector { + floating_ip: path.floating_ip, + project: query.project, + }; + let fip_lookup = + nexus.floating_ip_lookup(&opctx, floating_ip_selector)?; + let ip = nexus.floating_ip_detach(&opctx, fip_lookup).await?; + Ok(HttpResponseAccepted(ip)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -type IpPoolRangePaginationParams = PaginationParams; - -/// List ranges for IP pool -/// -/// Ranges are ordered by their first address. -#[endpoint { - method = GET, - path = "/v1/system/ip-pools/{pool}/ranges", - tags = ["system/networking"], -}] -async fn ip_pool_range_list( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let path = path_params.into_inner(); - let marker = match query.page { - WhichPage::First(_) => None, - WhichPage::Next(ref addr) => Some(addr), - }; - let pag_params = DataPageParams { - limit: rqctx.page_limit(&query)?, - direction: PaginationOrder::Ascending, - marker, - }; - let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; - let ranges = nexus - .ip_pool_list_ranges(&opctx, &pool_lookup, &pag_params) - .await? - .into_iter() - .map(|range| range.into()) - .collect(); - Ok(HttpResponseOk(ResultsPage::new( - ranges, - &EmptyScanParams {}, - |range: &IpPoolRange, _| { - IpNetwork::from(range.range.first_address()) - }, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Disks + + async fn disk_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let disks = nexus + .disk_list(&opctx, &project_lookup, &paginated_by) + .await? + .into_iter() + .map(|disk| disk.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + disks, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Add range to IP pool -/// -/// IPv6 ranges are not allowed yet. -#[endpoint { - method = POST, - path = "/v1/system/ip-pools/{pool}/ranges/add", - tags = ["system/networking"], -}] -async fn ip_pool_range_add( - rqctx: RequestContext, - path_params: Path, - range_params: TypedBody, -) -> Result, HttpError> { - let apictx = &rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let range = range_params.into_inner(); - let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; - let out = nexus.ip_pool_add_range(&opctx, &pool_lookup, &range).await?; - Ok(HttpResponseCreated(out.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // TODO-correctness See note about instance create. This should be async. + async fn disk_create( + rqctx: RequestContext, + query_params: Query, + new_disk: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let params = new_disk.into_inner(); + let project_lookup = nexus.project_lookup(&opctx, query)?; + let disk = nexus + .project_create_disk(&opctx, &project_lookup, ¶ms) + .await?; + Ok(HttpResponseCreated(disk.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Remove range from IP pool -#[endpoint { - method = POST, - path = "/v1/system/ip-pools/{pool}/ranges/remove", - tags = ["system/networking"], -}] -async fn ip_pool_range_remove( - rqctx: RequestContext, - path_params: Path, - range_params: TypedBody, -) -> Result { - let apictx = &rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let range = range_params.into_inner(); - let pool_lookup = nexus.ip_pool_lookup(&opctx, &path.pool)?; - nexus.ip_pool_delete_range(&opctx, &pool_lookup, &range).await?; - Ok(HttpResponseUpdatedNoContent()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn disk_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let disk_selector = params::DiskSelector { + disk: path.disk, + project: query.project, + }; + let (.., disk) = + nexus.disk_lookup(&opctx, disk_selector)?.fetch().await?; + Ok(HttpResponseOk(disk.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List IP ranges for the Oxide service pool -/// -/// Ranges are ordered by their first address. -#[endpoint { - method = GET, - path = "/v1/system/ip-pools-service/ranges", - tags = ["system/networking"], -}] -async fn ip_pool_service_range_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let marker = match query.page { - WhichPage::First(_) => None, - WhichPage::Next(ref addr) => Some(addr), - }; - let pag_params = DataPageParams { - limit: rqctx.page_limit(&query)?, - direction: PaginationOrder::Ascending, - marker, - }; - let ranges = nexus - .ip_pool_service_list_ranges(&opctx, &pag_params) - .await? - .into_iter() - .map(|range| range.into()) - .collect(); - Ok(HttpResponseOk(ResultsPage::new( - ranges, - &EmptyScanParams {}, - |range: &IpPoolRange, _| { - IpNetwork::from(range.range.first_address()) - }, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn disk_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let disk_selector = params::DiskSelector { + disk: path.disk, + project: query.project, + }; + let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; + nexus.project_delete_disk(&opctx, &disk_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Add IP range to Oxide service pool -/// -/// IPv6 ranges are not allowed yet. -#[endpoint { - method = POST, - path = "/v1/system/ip-pools-service/ranges/add", - tags = ["system/networking"], -}] -async fn ip_pool_service_range_add( - rqctx: RequestContext, - range_params: TypedBody, -) -> Result, HttpError> { - let apictx = &rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let range = range_params.into_inner(); - let out = nexus.ip_pool_service_add_range(&opctx, &range).await?; - Ok(HttpResponseCreated(out.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn disk_metrics_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query< + PaginationParams, + >, + selector_params: Query, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + let selector = selector_params.into_inner(); + let limit = rqctx.page_limit(&query)?; + let disk_selector = params::DiskSelector { + disk: path.disk, + project: selector.project, + }; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let (.., authz_disk) = nexus + .disk_lookup(&opctx, disk_selector)? + .lookup_for(authz::Action::Read) + .await?; -/// Remove IP range from Oxide service pool -#[endpoint { - method = POST, - path = "/v1/system/ip-pools-service/ranges/remove", - tags = ["system/networking"], -}] -async fn ip_pool_service_range_remove( - rqctx: RequestContext, - range_params: TypedBody, -) -> Result { - let apictx = &rqctx.context(); - let nexus = &apictx.context.nexus; - let range = range_params.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - nexus.ip_pool_service_delete_range(&opctx, &range).await?; - Ok(HttpResponseUpdatedNoContent()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let result = nexus + .select_timeseries( + &format!("crucible_upstairs:{}", path.metric), + &[&format!("upstairs_uuid=={}", authz_disk.id())], + query, + limit, + ) + .await?; -// Floating IP Addresses - -/// List floating IPs -#[endpoint { - method = GET, - path = "/v1/floating-ips", - tags = ["floating-ips"], -}] -async fn floating_ip_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let project_lookup = - nexus.project_lookup(&opctx, scan_params.selector.clone())?; - let ips = nexus - .floating_ips_list(&opctx, &project_lookup, &paginated_by) - .await?; - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - ips, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + Ok(HttpResponseOk(result)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create floating IP -#[endpoint { - method = POST, - path = "/v1/floating-ips", - tags = ["floating-ips"], -}] -async fn floating_ip_create( - rqctx: RequestContext, - query_params: Query, - floating_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let floating_params = floating_params.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_lookup = - nexus.project_lookup(&opctx, query_params.into_inner())?; - let ip = nexus - .floating_ip_create(&opctx, &project_lookup, floating_params) - .await?; - Ok(HttpResponseCreated(ip)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn disk_bulk_write_import_start( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + let disk_selector = params::DiskSelector { + disk: path.disk, + project: query.project, + }; + let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; -/// Update floating IP -#[endpoint { - method = PUT, - path = "/v1/floating-ips/{floating_ip}", - tags = ["floating-ips"], -}] -async fn floating_ip_update( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - updated_floating_ip: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let updated_floating_ip_params = updated_floating_ip.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let floating_ip_selector = params::FloatingIpSelector { - project: query.project, - floating_ip: path.floating_ip, + nexus.disk_manual_import_start(&opctx, &disk_lookup).await?; + + Ok(HttpResponseUpdatedNoContent()) }; - let floating_ip_lookup = - nexus.floating_ip_lookup(&opctx, floating_ip_selector)?; - let floating_ip = nexus - .floating_ip_update( - &opctx, - floating_ip_lookup, - updated_floating_ip_params, - ) - .await?; - Ok(HttpResponseOk(floating_ip)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete floating IP -#[endpoint { - method = DELETE, - path = "/v1/floating-ips/{floating_ip}", - tags = ["floating-ips"], -}] -async fn floating_ip_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let floating_ip_selector = params::FloatingIpSelector { - floating_ip: path.floating_ip, - project: query.project, + async fn disk_bulk_write_import( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + import_params: TypedBody, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let params = import_params.into_inner(); + + let disk_selector = params::DiskSelector { + disk: path.disk, + project: query.project, + }; + let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; + + nexus.disk_manual_import(&disk_lookup, params).await?; + + Ok(HttpResponseUpdatedNoContent()) }; - let fip_lookup = - nexus.floating_ip_lookup(&opctx, floating_ip_selector)?; - - nexus.floating_ip_delete(&opctx, fip_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch floating IP -#[endpoint { - method = GET, - path = "/v1/floating-ips/{floating_ip}", - tags = ["floating-ips"] -}] -async fn floating_ip_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let floating_ip_selector = params::FloatingIpSelector { - floating_ip: path.floating_ip, - project: query.project, - }; - let (.., fip) = nexus - .floating_ip_lookup(&opctx, floating_ip_selector)? - .fetch() - .await?; - Ok(HttpResponseOk(fip.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn disk_bulk_write_import_stop( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + + let disk_selector = params::DiskSelector { + disk: path.disk, + project: query.project, + }; + let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; -/// Attach floating IP -/// -/// Attach floating IP to an instance or other resource. -#[endpoint { - method = POST, - path = "/v1/floating-ips/{floating_ip}/attach", - tags = ["floating-ips"], -}] -async fn floating_ip_attach( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - target: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let floating_ip_selector = params::FloatingIpSelector { - floating_ip: path.floating_ip, - project: query.project, - }; - let ip = nexus - .floating_ip_attach( - &opctx, - floating_ip_selector, - target.into_inner(), - ) - .await?; - Ok(HttpResponseAccepted(ip)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + nexus.disk_manual_import_stop(&opctx, &disk_lookup).await?; -/// Detach floating IP -/// -// Detach floating IP from instance or other resource. -#[endpoint { - method = POST, - path = "/v1/floating-ips/{floating_ip}/detach", - tags = ["floating-ips"], -}] -async fn floating_ip_detach( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let floating_ip_selector = params::FloatingIpSelector { - floating_ip: path.floating_ip, - project: query.project, + Ok(HttpResponseUpdatedNoContent()) }; - let fip_lookup = - nexus.floating_ip_lookup(&opctx, floating_ip_selector)?; - let ip = nexus.floating_ip_detach(&opctx, fip_lookup).await?; - Ok(HttpResponseAccepted(ip)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -// Disks - -/// List disks -#[endpoint { - method = GET, - path = "/v1/disks", - tags = ["disks"], -}] -async fn disk_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let project_lookup = - nexus.project_lookup(&opctx, scan_params.selector.clone())?; - let disks = nexus - .disk_list(&opctx, &project_lookup, &paginated_by) - .await? - .into_iter() - .map(|disk| disk.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - disks, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -// TODO-correctness See note about instance create. This should be async. -/// Create a disk -#[endpoint { - method = POST, - path = "/v1/disks", - tags = ["disks"] -}] -async fn disk_create( - rqctx: RequestContext, - query_params: Query, - new_disk: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let params = new_disk.into_inner(); - let project_lookup = nexus.project_lookup(&opctx, query)?; - let disk = - nexus.project_create_disk(&opctx, &project_lookup, ¶ms).await?; - Ok(HttpResponseCreated(disk.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch disk -#[endpoint { - method = GET, - path = "/v1/disks/{disk}", - tags = ["disks"] -}] -async fn disk_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let disk_selector = - params::DiskSelector { disk: path.disk, project: query.project }; - let (.., disk) = - nexus.disk_lookup(&opctx, disk_selector)?.fetch().await?; - Ok(HttpResponseOk(disk.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn disk_finalize_import( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + finalize_params: TypedBody, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let params = finalize_params.into_inner(); + let disk_selector = params::DiskSelector { + disk: path.disk, + project: query.project, + }; + let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; -/// Delete disk -#[endpoint { - method = DELETE, - path = "/v1/disks/{disk}", - tags = ["disks"], -}] -async fn disk_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let disk_selector = - params::DiskSelector { disk: path.disk, project: query.project }; - let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; - nexus.project_delete_disk(&opctx, &disk_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + nexus.disk_finalize_import(&opctx, &disk_lookup, ¶ms).await?; -#[derive(Display, Serialize, Deserialize, JsonSchema)] -#[display(style = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum DiskMetricName { - Activated, - Flush, - Read, - ReadBytes, - Write, - WriteBytes, -} + Ok(HttpResponseUpdatedNoContent()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -#[derive(Serialize, Deserialize, JsonSchema)] -struct DiskMetricsPath { - disk: NameOrId, - metric: DiskMetricName, -} + // Instances + + async fn instance_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let instances = nexus + .instance_list(&opctx, &project_lookup, &paginated_by) + .await? + .into_iter() + .map(|i| i.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + instances, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch disk metrics -#[endpoint { - method = GET, - path = "/v1/disks/{disk}/metrics/{metric}", - tags = ["disks"], -}] -async fn disk_metrics_list( - rqctx: RequestContext, - path_params: Path, - query_params: Query< - PaginationParams, - >, - selector_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { + async fn instance_create( + rqctx: RequestContext, + query_params: Query, + new_instance: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - - let selector = selector_params.into_inner(); - let limit = rqctx.page_limit(&query)?; - let disk_selector = - params::DiskSelector { disk: path.disk, project: selector.project }; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let (.., authz_disk) = nexus - .disk_lookup(&opctx, disk_selector)? - .lookup_for(authz::Action::Read) - .await?; - - let result = nexus - .select_timeseries( - &format!("crucible_upstairs:{}", path.metric), - &[&format!("upstairs_uuid=={}", authz_disk.id())], - query, - limit, - ) - .await?; - - Ok(HttpResponseOk(result)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let project_selector = query_params.into_inner(); + let new_instance_params = &new_instance.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_lookup = + nexus.project_lookup(&opctx, project_selector)?; + let instance = nexus + .project_create_instance( + &opctx, + &project_lookup, + &new_instance_params, + ) + .await?; + Ok(HttpResponseCreated(instance.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Start importing blocks into disk -/// -/// Start the process of importing blocks into a disk -#[endpoint { - method = POST, - path = "/v1/disks/{disk}/bulk-write-start", - tags = ["disks"], -}] -async fn disk_bulk_write_import_start( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn instance_view( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); let query = query_params.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let (.., authz_instance) = + instance_lookup.lookup_for(authz::Action::Read).await?; + let instance_and_vmm = nexus + .datastore() + .instance_fetch_with_vmm(&opctx, &authz_instance) + .await?; + Ok(HttpResponseOk(instance_and_vmm.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let disk_selector = - params::DiskSelector { disk: path.disk, project: query.project }; - let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; - - nexus.disk_manual_import_start(&opctx, &disk_lookup).await?; - - Ok(HttpResponseUpdatedNoContent()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Import blocks into disk -#[endpoint { - method = POST, - path = "/v1/disks/{disk}/bulk-write", - tags = ["disks"], -}] -async fn disk_bulk_write_import( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - import_params: TypedBody, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn instance_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); let query = query_params.into_inner(); - let params = import_params.into_inner(); - - let disk_selector = - params::DiskSelector { disk: path.disk, project: query.project }; - let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; - - nexus.disk_manual_import(&disk_lookup, params).await?; - - Ok(HttpResponseUpdatedNoContent()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + nexus.project_destroy_instance(&opctx, &instance_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Stop importing blocks into disk -/// -/// Stop the process of importing blocks into a disk -#[endpoint { - method = POST, - path = "/v1/disks/{disk}/bulk-write-stop", - tags = ["disks"], -}] -async fn disk_bulk_write_import_stop( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn instance_reboot( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); let query = query_params.into_inner(); + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let instance = + nexus.instance_reboot(&opctx, &instance_lookup).await?; + Ok(HttpResponseAccepted(instance.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let disk_selector = - params::DiskSelector { disk: path.disk, project: query.project }; - let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; - - nexus.disk_manual_import_stop(&opctx, &disk_lookup).await?; - - Ok(HttpResponseUpdatedNoContent()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Confirm disk block import completion -#[endpoint { - method = POST, - path = "/v1/disks/{disk}/finalize", - tags = ["disks"], -}] -async fn disk_finalize_import( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - finalize_params: TypedBody, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn instance_start( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); let query = query_params.into_inner(); - let params = finalize_params.into_inner(); - let disk_selector = - params::DiskSelector { disk: path.disk, project: query.project }; - let disk_lookup = nexus.disk_lookup(&opctx, disk_selector)?; - - nexus.disk_finalize_import(&opctx, &disk_lookup, ¶ms).await?; - - Ok(HttpResponseUpdatedNoContent()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -// Instances - -/// List instances -#[endpoint { - method = GET, - path = "/v1/instances", - tags = ["instances"], -}] -async fn instance_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_lookup = - nexus.project_lookup(&opctx, scan_params.selector.clone())?; - let instances = nexus - .instance_list(&opctx, &project_lookup, &paginated_by) - .await? - .into_iter() - .map(|i| i.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - instances, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Create instance -#[endpoint { - method = POST, - path = "/v1/instances", - tags = ["instances"], -}] -async fn instance_create( - rqctx: RequestContext, - query_params: Query, - new_instance: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let project_selector = query_params.into_inner(); - let new_instance_params = &new_instance.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_lookup = nexus.project_lookup(&opctx, project_selector)?; - let instance = nexus - .project_create_instance( - &opctx, - &project_lookup, - &new_instance_params, - ) - .await?; - Ok(HttpResponseCreated(instance.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Fetch instance -#[endpoint { - method = GET, - path = "/v1/instances/{instance}", - tags = ["instances"], -}] -async fn instance_view( - rqctx: RequestContext, - query_params: Query, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let instance_selector = params::InstanceSelector { project: query.project, instance: path.instance, }; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let (.., authz_instance) = - instance_lookup.lookup_for(authz::Action::Read).await?; - let instance_and_vmm = nexus - .datastore() - .instance_fetch_with_vmm(&opctx, &authz_instance) - .await?; - Ok(HttpResponseOk(instance_and_vmm.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Delete instance -#[endpoint { - method = DELETE, - path = "/v1/instances/{instance}", - tags = ["instances"], -}] -async fn instance_delete( - rqctx: RequestContext, - query_params: Query, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let instance_selector = params::InstanceSelector { - project: query.project, - instance: path.instance, - }; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - nexus.project_destroy_instance(&opctx, &instance_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Reboot an instance -#[endpoint { - method = POST, - path = "/v1/instances/{instance}/reboot", - tags = ["instances"], -}] -async fn instance_reboot( - rqctx: RequestContext, - query_params: Query, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let instance_selector = params::InstanceSelector { - project: query.project, - instance: path.instance, - }; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let instance = nexus.instance_reboot(&opctx, &instance_lookup).await?; - Ok(HttpResponseAccepted(instance.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Boot instance -#[endpoint { - method = POST, - path = "/v1/instances/{instance}/start", - tags = ["instances"], -}] -async fn instance_start( - rqctx: RequestContext, - query_params: Query, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let instance_selector = params::InstanceSelector { - project: query.project, - instance: path.instance, - }; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let instance = nexus.instance_start(&opctx, &instance_lookup).await?; - Ok(HttpResponseAccepted(instance.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} - -/// Stop instance -#[endpoint { - method = POST, - path = "/v1/instances/{instance}/stop", - tags = ["instances"], -}] -async fn instance_stop( - rqctx: RequestContext, - query_params: Query, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let instance_selector = params::InstanceSelector { - project: query.project, - instance: path.instance, - }; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let instance = nexus.instance_stop(&opctx, &instance_lookup).await?; - Ok(HttpResponseAccepted(instance.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let instance = + nexus.instance_start(&opctx, &instance_lookup).await?; + Ok(HttpResponseAccepted(instance.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch instance serial console -#[endpoint { - method = GET, - path = "/v1/instances/{instance}/serial-console", - tags = ["instances"], -}] -async fn instance_serial_console( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn instance_stop( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); let query = query_params.into_inner(); let instance_selector = params::InstanceSelector { - project: query.project.clone(), + project: query.project, instance: path.instance, }; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let data = nexus - .instance_serial_console_data(&opctx, &instance_lookup, &query) - .await?; - Ok(HttpResponseOk(data)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let instance = + nexus.instance_stop(&opctx, &instance_lookup).await?; + Ok(HttpResponseAccepted(instance.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Stream instance serial console -#[channel { - protocol = WEBSOCKETS, - path = "/v1/instances/{instance}/serial-console/stream", - tags = ["instances"], -}] -async fn instance_serial_console_stream( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - conn: WebsocketConnection, -) -> WebsocketChannelResult { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let instance_selector = params::InstanceSelector { - project: query.project.clone(), - instance: path.instance, - }; - let mut client_stream = WebSocketStream::from_raw_socket( - conn.into_inner(), - WebSocketRole::Server, - None, - ) - .await; - match nexus.instance_lookup(&opctx, instance_selector) { - Ok(instance_lookup) => { - nexus - .instance_serial_console_stream( - &opctx, - client_stream, - &instance_lookup, - &query, - ) + async fn instance_serial_console( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let instance_selector = params::InstanceSelector { + project: query.project.clone(), + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let data = nexus + .instance_serial_console_data(&opctx, &instance_lookup, &query) .await?; - Ok(()) - } - Err(e) => { - let _ = client_stream - .close(Some(CloseFrame { - code: CloseCode::Error, - reason: e.to_string().into(), - })) - .await - .is_ok(); - Err(e.into()) - } + Ok(HttpResponseOk(data)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await } -} -/// List SSH public keys for instance -/// -/// List SSH public keys injected via cloud-init during instance creation. Note -/// that this list is a snapshot in time and will not reflect updates made after -/// the instance is created. -#[endpoint { - method = GET, - path = "/v1/instances/{instance}/ssh-public-keys", - tags = ["instances"], -}] -async fn instance_ssh_public_key_list( - rqctx: RequestContext, - path_params: Path, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { + async fn instance_serial_console_stream( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + conn: WebsocketConnection, + ) -> WebsocketChannelResult { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let instance_selector = params::InstanceSelector { - project: scan_params.selector.project.clone(), + project: query.project.clone(), instance: path.instance, }; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let ssh_keys = nexus - .instance_ssh_keys_list(&opctx, &instance_lookup, &paginated_by) - .await? - .into_iter() - .map(|k| k.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - ssh_keys, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let mut client_stream = WebSocketStream::from_raw_socket( + conn.into_inner(), + WebSocketRole::Server, + None, + ) + .await; + match nexus.instance_lookup(&opctx, instance_selector) { + Ok(instance_lookup) => { + nexus + .instance_serial_console_stream( + &opctx, + client_stream, + &instance_lookup, + &query, + ) + .await?; + Ok(()) + } + Err(e) => { + let _ = client_stream + .close(Some(CloseFrame { + code: CloseCode::Error, + reason: e.to_string().into(), + })) + .await + .is_ok(); + Err(e.into()) + } + } + } -/// List disks for instance -#[endpoint { - method = GET, - path = "/v1/instances/{instance}/disks", - tags = ["instances"], -}] -async fn instance_disk_list( - rqctx: RequestContext, - query_params: Query>, - path_params: Path, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let instance_selector = params::InstanceSelector { - project: scan_params.selector.project.clone(), - instance: path.instance, + async fn instance_ssh_public_key_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let instance_selector = params::InstanceSelector { + project: scan_params.selector.project.clone(), + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let ssh_keys = nexus + .instance_ssh_keys_list(&opctx, &instance_lookup, &paginated_by) + .await? + .into_iter() + .map(|k| k.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + ssh_keys, + &marker_for_name_or_id, + )?)) }; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let disks = nexus - .instance_list_disks(&opctx, &instance_lookup, &paginated_by) - .await? - .into_iter() - .map(|d| d.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - disks, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Attach disk to instance -#[endpoint { - method = POST, - path = "/v1/instances/{instance}/disks/attach", - tags = ["instances"], -}] -async fn instance_disk_attach( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - disk_to_attach: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let disk = disk_to_attach.into_inner().disk; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let instance_selector = params::InstanceSelector { - project: query.project, - instance: path.instance, + async fn instance_disk_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + path_params: Path, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let instance_selector = params::InstanceSelector { + project: scan_params.selector.project.clone(), + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let disks = nexus + .instance_list_disks(&opctx, &instance_lookup, &paginated_by) + .await? + .into_iter() + .map(|d| d.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + disks, + &marker_for_name_or_id, + )?)) }; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let disk = - nexus.instance_attach_disk(&opctx, &instance_lookup, disk).await?; - Ok(HttpResponseAccepted(disk.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Detach disk from instance -#[endpoint { - method = POST, - path = "/v1/instances/{instance}/disks/detach", - tags = ["instances"], -}] -async fn instance_disk_detach( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - disk_to_detach: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn instance_disk_attach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + disk_to_attach: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; let path = path_params.into_inner(); let query = query_params.into_inner(); - let disk = disk_to_detach.into_inner().disk; - let instance_selector = params::InstanceSelector { - project: query.project, - instance: path.instance, + let disk = disk_to_attach.into_inner().disk; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let disk = nexus + .instance_attach_disk(&opctx, &instance_lookup, disk) + .await?; + Ok(HttpResponseAccepted(disk.into())) }; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let disk = - nexus.instance_detach_disk(&opctx, &instance_lookup, disk).await?; - Ok(HttpResponseAccepted(disk.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Certificates - -/// List certificates for external endpoints -/// -/// Returns a list of TLS certificates used for the external API (for the -/// current Silo). These are sorted by creation date, with the most recent -/// certificates appearing first. -#[endpoint { - method = GET, - path = "/v1/certificates", - tags = ["silos"], -}] -async fn certificate_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let certs = nexus - .certificates_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(|d| d.try_into()) - .collect::, Error>>()?; - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - certs, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn instance_disk_detach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + disk_to_detach: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let disk = disk_to_detach.into_inner().disk; + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let disk = nexus + .instance_detach_disk(&opctx, &instance_lookup, disk) + .await?; + Ok(HttpResponseAccepted(disk.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create new system-wide x.509 certificate -/// -/// This certificate is automatically used by the Oxide Control plane to serve -/// external connections. -#[endpoint { - method = POST, - path = "/v1/certificates", - tags = ["silos"] -}] -async fn certificate_create( - rqctx: RequestContext, - new_cert: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let new_cert_params = new_cert.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let cert = nexus.certificate_create(&opctx, new_cert_params).await?; - Ok(HttpResponseCreated(cert.try_into()?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Certificates + + async fn certificate_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let certs = nexus + .certificates_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|d| d.try_into()) + .collect::, Error>>()?; + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + certs, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Path parameters for Certificate requests -#[derive(Deserialize, JsonSchema)] -struct CertificatePathParam { - certificate: NameOrId, -} + async fn certificate_create( + rqctx: RequestContext, + new_cert: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let new_cert_params = new_cert.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let cert = + nexus.certificate_create(&opctx, new_cert_params).await?; + Ok(HttpResponseCreated(cert.try_into()?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch certificate -/// -/// Returns the details of a specific certificate -#[endpoint { - method = GET, - path = "/v1/certificates/{certificate}", - tags = ["silos"], -}] -async fn certificate_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let (.., cert) = - nexus.certificate_lookup(&opctx, &path.certificate).fetch().await?; - Ok(HttpResponseOk(cert.try_into()?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn certificate_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let (.., cert) = nexus + .certificate_lookup(&opctx, &path.certificate) + .fetch() + .await?; + Ok(HttpResponseOk(cert.try_into()?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete certificate -/// -/// Permanently delete a certificate. This operation cannot be undone. -#[endpoint { - method = DELETE, - path = "/v1/certificates/{certificate}", - tags = ["silos"], -}] -async fn certificate_delete( - rqctx: RequestContext, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - nexus - .certificate_delete( - &opctx, - nexus.certificate_lookup(&opctx, &path.certificate), - ) - .await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn certificate_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + nexus + .certificate_delete( + &opctx, + nexus.certificate_lookup(&opctx, &path.certificate), + ) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create address lot -#[endpoint { - method = POST, - path = "/v1/system/networking/address-lot", - tags = ["system/networking"], -}] -async fn networking_address_lot_create( - rqctx: RequestContext, - new_address_lot: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let params = new_address_lot.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus.address_lot_create(&opctx, params).await?; - - let lot: AddressLot = result.lot.into(); - let blocks: Vec = - result.blocks.iter().map(|b| b.clone().into()).collect(); - - Ok(HttpResponseCreated(AddressLotCreateResponse { lot, blocks })) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_address_lot_create( + rqctx: RequestContext, + new_address_lot: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let params = new_address_lot.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.address_lot_create(&opctx, params).await?; + + let lot: AddressLot = result.lot.into(); + let blocks: Vec = + result.blocks.iter().map(|b| b.clone().into()).collect(); + + Ok(HttpResponseCreated(AddressLotCreateResponse { lot, blocks })) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete address lot -#[endpoint { - method = DELETE, - path = "/v1/system/networking/address-lot/{address_lot}", - tags = ["system/networking"], -}] -async fn networking_address_lot_delete( - rqctx: RequestContext, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let address_lot_lookup = - nexus.address_lot_lookup(&opctx, path.address_lot)?; - nexus.address_lot_delete(&opctx, &address_lot_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_address_lot_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let address_lot_lookup = + nexus.address_lot_lookup(&opctx, path.address_lot)?; + nexus.address_lot_delete(&opctx, &address_lot_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List address lots -#[endpoint { - method = GET, - path = "/v1/system/networking/address-lot", - tags = ["system/networking"], -}] -async fn networking_address_lot_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let lots = nexus - .address_lot_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - lots, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_address_lot_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let lots = nexus + .address_lot_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + lots, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List blocks in address lot -#[endpoint { - method = GET, - path = "/v1/system/networking/address-lot/{address_lot}/blocks", - tags = ["system/networking"], -}] -async fn networking_address_lot_block_list( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let path = path_params.into_inner(); - let pagparams = data_page_params_for(&rqctx, &query)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let address_lot_lookup = - nexus.address_lot_lookup(&opctx, path.address_lot)?; - let blocks = nexus - .address_lot_block_list(&opctx, &address_lot_lookup, &pagparams) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(ScanById::results_page( - &query, - blocks, - &|_, x: &AddressLotBlock| x.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_address_lot_block_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let path = path_params.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let address_lot_lookup = + nexus.address_lot_lookup(&opctx, path.address_lot)?; + let blocks = nexus + .address_lot_block_list(&opctx, &address_lot_lookup, &pagparams) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanById::results_page( + &query, + blocks, + &|_, x: &AddressLotBlock| x.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create loopback address -#[endpoint { - method = POST, - path = "/v1/system/networking/loopback-address", - tags = ["system/networking"], -}] -async fn networking_loopback_address_create( - rqctx: RequestContext, - new_loopback_address: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let params = new_loopback_address.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus.loopback_address_create(&opctx, params).await?; + async fn networking_loopback_address_create( + rqctx: RequestContext, + new_loopback_address: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let params = new_loopback_address.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.loopback_address_create(&opctx, params).await?; + + let addr: LoopbackAddress = result.into(); + + Ok(HttpResponseCreated(addr)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let addr: LoopbackAddress = result.into(); + async fn networking_loopback_address_delete( + rqctx: RequestContext, + path: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let addr = match IpNetwork::new(path.address, path.subnet_mask) { + Ok(addr) => Ok(addr), + Err(_) => Err(HttpError::for_bad_request( + None, + "invalid ip address".into(), + )), + }?; + nexus + .loopback_address_delete( + &opctx, + path.rack_id, + path.switch_location.into(), + addr.into(), + ) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - Ok(HttpResponseCreated(addr)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_loopback_address_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let addrs = nexus + .loopback_address_list(&opctx, &pagparams) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanById::results_page( + &query, + addrs, + &|_, x: &LoopbackAddress| x.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -#[derive(Serialize, Deserialize, JsonSchema)] -pub struct LoopbackAddressPath { - /// The rack to use when selecting the loopback address. - pub rack_id: Uuid, + async fn networking_switch_port_settings_create( + rqctx: RequestContext, + new_settings: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let params = new_settings.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let result = + nexus.switch_port_settings_post(&opctx, params).await?; + + let settings: SwitchPortSettingsView = result.into(); + Ok(HttpResponseCreated(settings)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - /// The switch location to use when selecting the loopback address. - pub switch_location: Name, + async fn networking_switch_port_settings_delete( + rqctx: RequestContext, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let selector = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + nexus.switch_port_settings_delete(&opctx, &selector).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - /// The IP address and subnet mask to use when selecting the loopback - /// address. - pub address: IpAddr, + async fn networking_switch_port_settings_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let settings = nexus + .switch_port_settings_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + settings, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - /// The IP address and subnet mask to use when selecting the loopback - /// address. - pub subnet_mask: u8, -} + async fn networking_switch_port_settings_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = path_params.into_inner().port; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let settings = + nexus.switch_port_settings_get(&opctx, &query).await?; + Ok(HttpResponseOk(settings.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete loopback address -#[endpoint { - method = DELETE, - path = "/v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask}", - tags = ["system/networking"], -}] -async fn networking_loopback_address_delete( - rqctx: RequestContext, - path: Path, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let addr = match IpNetwork::new(path.address, path.subnet_mask) { - Ok(addr) => Ok(addr), - Err(_) => Err(HttpError::for_bad_request( - None, - "invalid ip address".into(), - )), - }?; - nexus - .loopback_address_delete( - &opctx, - path.rack_id, - path.switch_location.clone(), - addr.into(), - ) - .await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_switch_port_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let addrs = nexus + .switch_port_list(&opctx, &pagparams) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanById::results_page( + &query, + addrs, + &|_, x: &SwitchPort| x.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List loopback addresses -#[endpoint { - method = GET, - path = "/v1/system/networking/loopback-address", - tags = ["system/networking"], -}] -async fn networking_loopback_address_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pagparams = data_page_params_for(&rqctx, &query)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let addrs = nexus - .loopback_address_list(&opctx, &pagparams) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(ScanById::results_page( - &query, - addrs, - &|_, x: &LoopbackAddress| x.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_switch_port_status( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + Ok(HttpResponseOk( + nexus + .switch_port_status( + &opctx, + query.switch_location, + path.port, + ) + .await?, + )) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create switch port settings -#[endpoint { - method = POST, - path = "/v1/system/networking/switch-port-settings", - tags = ["system/networking"], -}] -async fn networking_switch_port_settings_create( - rqctx: RequestContext, - new_settings: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let params = new_settings.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus.switch_port_settings_post(&opctx, params).await?; - - let settings: SwitchPortSettingsView = result.into(); - Ok(HttpResponseCreated(settings)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_switch_port_apply_settings( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + settings_body: TypedBody, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let port = path_params.into_inner().port; + let query = query_params.into_inner(); + let settings = settings_body.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + nexus + .switch_port_apply_settings(&opctx, &port, &query, &settings) + .await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete switch port settings -#[endpoint { - method = DELETE, - path = "/v1/system/networking/switch-port-settings", - tags = ["system/networking"], -}] -async fn networking_switch_port_settings_delete( - rqctx: RequestContext, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let selector = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - nexus.switch_port_settings_delete(&opctx, &selector).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_switch_port_clear_settings( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let port = path_params.into_inner().port; + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + nexus.switch_port_clear_settings(&opctx, &port, &query).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List switch port settings -#[endpoint { - method = GET, - path = "/v1/system/networking/switch-port-settings", - tags = ["system/networking"], -}] -async fn networking_switch_port_settings_list( - rqctx: RequestContext, - query_params: Query< - PaginatedByNameOrId, - >, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let settings = nexus - .switch_port_settings_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - settings, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bgp_config_create( + rqctx: RequestContext, + config: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let config = config.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.bgp_config_create(&opctx, &config).await?; + Ok(HttpResponseCreated::(result.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Get information about switch port -#[endpoint { - method = GET, - path = "/v1/system/networking/switch-port-settings/{port}", - tags = ["system/networking"], -}] -async fn networking_switch_port_settings_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = path_params.into_inner().port; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let settings = nexus.switch_port_settings_get(&opctx, &query).await?; - Ok(HttpResponseOk(settings.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bgp_config_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let configs = nexus + .bgp_config_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + configs, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List switch ports -#[endpoint { - method = GET, - path = "/v1/system/hardware/switch-port", - tags = ["system/hardware"], -}] -async fn networking_switch_port_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pagparams = data_page_params_for(&rqctx, &query)?; + //TODO pagination? the normal by-name/by-id stuff does not work here + async fn networking_bgp_status( + rqctx: RequestContext, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let addrs = nexus - .switch_port_list(&opctx, &pagparams) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(ScanById::results_page( - &query, - addrs, - &|_, x: &SwitchPort| x.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let handler = async { + let nexus = &apictx.context.nexus; + let result = nexus.bgp_peer_status(&opctx).await?; + Ok(HttpResponseOk(result)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Get switch port status -#[endpoint { - method = GET, - path = "/v1/system/hardware/switch-port/{port}/status", - tags = ["system/hardware"], -}] -async fn networking_switch_port_status( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let path = path_params.into_inner(); + //TODO pagination? the normal by-name/by-id stuff does not work here + async fn networking_bgp_exported( + rqctx: RequestContext, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - Ok(HttpResponseOk( - nexus - .switch_port_status(&opctx, query.switch_location, path.port) - .await?, - )) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let handler = async { + let nexus = &apictx.context.nexus; + let result = nexus.bgp_exported(&opctx).await?; + Ok(HttpResponseOk(result)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Apply switch port settings -#[endpoint { - method = POST, - path = "/v1/system/hardware/switch-port/{port}/settings", - tags = ["system/hardware"], -}] -async fn networking_switch_port_apply_settings( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - settings_body: TypedBody, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let port = path_params.into_inner().port; - let query = query_params.into_inner(); - let settings = settings_body.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - nexus - .switch_port_apply_settings(&opctx, &port, &query, &settings) - .await?; - Ok(HttpResponseUpdatedNoContent {}) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bgp_message_history( + rqctx: RequestContext, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let handler = async { + let nexus = &apictx.context.nexus; + let sel = query_params.into_inner(); + let result = nexus.bgp_message_history(&opctx, &sel).await?; + Ok(HttpResponseOk(AggregateBgpMessageHistory::new(result))) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Clear switch port settings -#[endpoint { - method = DELETE, - path = "/v1/system/hardware/switch-port/{port}/settings", - tags = ["system/hardware"], -}] -async fn networking_switch_port_clear_settings( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let port = path_params.into_inner().port; - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - nexus.switch_port_clear_settings(&opctx, &port, &query).await?; - Ok(HttpResponseUpdatedNoContent {}) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + //TODO pagination? the normal by-name/by-id stuff does not work here + async fn networking_bgp_imported_routes_ipv4( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let handler = async { + let nexus = &apictx.context.nexus; + let sel = query_params.into_inner(); + let result = nexus.bgp_imported_routes_ipv4(&opctx, &sel).await?; + Ok(HttpResponseOk(result)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create new BGP configuration -#[endpoint { - method = POST, - path = "/v1/system/networking/bgp", - tags = ["system/networking"], -}] -async fn networking_bgp_config_create( - rqctx: RequestContext, - config: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let config = config.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus.bgp_config_create(&opctx, &config).await?; - Ok(HttpResponseCreated::(result.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bgp_config_delete( + rqctx: RequestContext, + sel: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let sel = sel.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + nexus.bgp_config_delete(&opctx, &sel).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List BGP configurations -#[endpoint { - method = GET, - path = "/v1/system/networking/bgp", - tags = ["system/networking"], -}] -async fn networking_bgp_config_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let configs = nexus - .bgp_config_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - configs, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bgp_announce_set_update( + rqctx: RequestContext, + config: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let config = config.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.bgp_update_announce_set(&opctx, &config).await?; + Ok(HttpResponseCreated::(result.0.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -//TODO pagination? the normal by-name/by-id stuff does not work here -/// Get BGP peer status -#[endpoint { - method = GET, - path = "/v1/system/networking/bgp-status", - tags = ["system/networking"], -}] -async fn networking_bgp_status( - rqctx: RequestContext, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let handler = async { - let nexus = &apictx.context.nexus; - let result = nexus.bgp_peer_status(&opctx).await?; - Ok(HttpResponseOk(result)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bgp_announce_set_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus + .bgp_announce_set_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + Ok(HttpResponseOk(result)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -//TODO pagination? the normal by-name/by-id stuff does not work here -/// Get BGP exported routes -#[endpoint { - method = GET, - path = "/v1/system/networking/bgp-exported", - tags = ["system/networking"], -}] -async fn networking_bgp_exported( - rqctx: RequestContext, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let handler = async { - let nexus = &apictx.context.nexus; - let result = nexus.bgp_exported(&opctx).await?; - Ok(HttpResponseOk(result)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bgp_announce_set_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let sel = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + nexus.bgp_delete_announce_set(&opctx, &sel).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Get BGP router message history -#[endpoint { - method = GET, - path = "/v1/system/networking/bgp-message-history", - tags = ["system/networking"], -}] -async fn networking_bgp_message_history( - rqctx: RequestContext, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let handler = async { - let nexus = &apictx.context.nexus; - let sel = query_params.into_inner(); - let result = nexus.bgp_message_history(&opctx, &sel).await?; - Ok(HttpResponseOk(AggregateBgpMessageHistory::new(result))) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bgp_announcement_list( + rqctx: RequestContext, + path_params: Path, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let sel = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + let result = nexus + .bgp_announcement_list(&opctx, &sel) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); -//TODO pagination? the normal by-name/by-id stuff does not work here -/// Get imported IPv4 BGP routes -#[endpoint { - method = GET, - path = "/v1/system/networking/bgp-routes-ipv4", - tags = ["system/networking"], -}] -async fn networking_bgp_imported_routes_ipv4( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let handler = async { - let nexus = &apictx.context.nexus; - let sel = query_params.into_inner(); - let result = nexus.bgp_imported_routes_ipv4(&opctx, &sel).await?; - Ok(HttpResponseOk(result)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + Ok(HttpResponseOk(result)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete BGP configuration -#[endpoint { - method = DELETE, - path = "/v1/system/networking/bgp", - tags = ["system/networking"], -}] -async fn networking_bgp_config_delete( - rqctx: RequestContext, - sel: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let sel = sel.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - nexus.bgp_config_delete(&opctx, &sel).await?; - Ok(HttpResponseUpdatedNoContent {}) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bfd_enable( + rqctx: RequestContext, + session: TypedBody, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + nexus.bfd_enable(&opctx, session.into_inner()).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update BGP announce set -/// -/// If the announce set exists, this endpoint replaces the existing announce -/// set with the one specified. -#[endpoint { - method = PUT, - path = "/v1/system/networking/bgp-announce-set", - tags = ["system/networking"], -}] -async fn networking_bgp_announce_set_update( - rqctx: RequestContext, - config: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let config = config.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus.bgp_update_announce_set(&opctx, &config).await?; - Ok(HttpResponseCreated::(result.0.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bfd_disable( + rqctx: RequestContext, + session: TypedBody, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + nexus.bfd_disable(&opctx, session.into_inner()).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List BGP announce sets -#[endpoint { - method = GET, - path = "/v1/system/networking/bgp-announce-set", - tags = ["system/networking"], -}] -async fn networking_bgp_announce_set_list( - rqctx: RequestContext, - query_params: Query< - PaginatedByNameOrId, - >, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus - .bgp_announce_set_list(&opctx, &paginated_by) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - Ok(HttpResponseOk(result)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_bfd_status( + rqctx: RequestContext, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + let status = nexus.bfd_status(&opctx).await?; + Ok(HttpResponseOk(status)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete BGP announce set -#[endpoint { - method = DELETE, - path = "/v1/system/networking/bgp-announce-set/{name_or_id}", - tags = ["system/networking"], -}] -async fn networking_bgp_announce_set_delete( - rqctx: RequestContext, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let sel = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - nexus.bgp_delete_announce_set(&opctx, &sel).await?; - Ok(HttpResponseUpdatedNoContent {}) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn networking_allow_list_view( + rqctx: RequestContext, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + nexus + .allow_list_view(&opctx) + .await + .map(HttpResponseOk) + .map_err(HttpError::from) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// TODO: is pagination necessary here? How large do we expect the list of -// announcements to become in real usage? -/// Get originated routes for a specified BGP announce set -#[endpoint { - method = GET, - path = "/v1/system/networking/bgp-announce-set/{name_or_id}/announcement", - tags = ["system/networking"], -}] -async fn networking_bgp_announcement_list( - rqctx: RequestContext, - path_params: Path, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let sel = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn networking_allow_list_update( + rqctx: RequestContext, + params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let server_kind = apictx.kind; + let params = params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let remote_addr = rqctx.request.remote_addr().ip(); + nexus + .allow_list_upsert(&opctx, remote_addr, server_kind, params) + .await + .map(HttpResponseOk) + .map_err(HttpError::from) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let result = nexus - .bgp_announcement_list(&opctx, &sel) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(result)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Images + + async fn image_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let parent_lookup = match scan_params.selector.project.clone() { + Some(project) => { + let project_lookup = nexus.project_lookup( + &opctx, + params::ProjectSelector { project }, + )?; + ImageParentLookup::Project(project_lookup) + } + None => { + let silo_lookup = nexus.current_silo_lookup(&opctx)?; + ImageParentLookup::Silo(silo_lookup) + } + }; + let images = nexus + .image_list(&opctx, &parent_lookup, &paginated_by) + .await? + .into_iter() + .map(|d| d.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + images, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Enable a BFD session -#[endpoint { - method = POST, - path = "/v1/system/networking/bfd-enable", - tags = ["system/networking"], -}] -async fn networking_bfd_enable( - rqctx: RequestContext, - session: TypedBody, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; - nexus.bfd_enable(&opctx, session.into_inner()).await?; - Ok(HttpResponseUpdatedNoContent {}) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn image_create( + rqctx: RequestContext, + query_params: Query, + new_image: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let params = &new_image.into_inner(); + let parent_lookup = match query.project.clone() { + Some(project) => { + let project_lookup = nexus.project_lookup( + &opctx, + params::ProjectSelector { project }, + )?; + ImageParentLookup::Project(project_lookup) + } + None => { + let silo_lookup = nexus.current_silo_lookup(&opctx)?; + ImageParentLookup::Silo(silo_lookup) + } + }; + let image = + nexus.image_create(&opctx, &parent_lookup, ¶ms).await?; + Ok(HttpResponseCreated(image.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Disable a BFD session -#[endpoint { - method = POST, - path = "/v1/system/networking/bfd-disable", - tags = ["system/networking"], -}] -async fn networking_bfd_disable( - rqctx: RequestContext, - session: TypedBody, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; - nexus.bfd_disable(&opctx, session.into_inner()).await?; - Ok(HttpResponseUpdatedNoContent {}) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn image_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let image: nexus_db_model::Image = match nexus + .image_lookup( + &opctx, + params::ImageSelector { + image: path.image, + project: query.project, + }, + ) + .await? + { + ImageLookup::ProjectImage(image) => { + let (.., db_image) = image.fetch().await?; + db_image.into() + } + ImageLookup::SiloImage(image) => { + let (.., db_image) = image.fetch().await?; + db_image.into() + } + }; + Ok(HttpResponseOk(image.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Get BFD status -#[endpoint { - method = GET, - path = "/v1/system/networking/bfd-status", - tags = ["system/networking"], -}] -async fn networking_bfd_status( - rqctx: RequestContext, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; - let status = nexus.bfd_status(&opctx).await?; - Ok(HttpResponseOk(status)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn image_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let image_lookup = nexus + .image_lookup( + &opctx, + params::ImageSelector { + image: path.image, + project: query.project, + }, + ) + .await?; + nexus.image_delete(&opctx, &image_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Get user-facing services IP allowlist -#[endpoint { - method = GET, - path = "/v1/system/networking/allow-list", - tags = ["system/networking"], -}] -async fn networking_allow_list_view( - rqctx: RequestContext, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - nexus - .allow_list_view(&opctx) - .await - .map(HttpResponseOk) - .map_err(HttpError::from) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn image_promote( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let image_lookup = nexus + .image_lookup( + &opctx, + params::ImageSelector { + image: path.image, + project: query.project, + }, + ) + .await?; + let image = nexus.image_promote(&opctx, &image_lookup).await?; + Ok(HttpResponseAccepted(image.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update user-facing services IP allowlist -#[endpoint { - method = PUT, - path = "/v1/system/networking/allow-list", - tags = ["system/networking"], -}] -async fn networking_allow_list_update( - rqctx: RequestContext, - params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let server_kind = apictx.kind; - let params = params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let remote_addr = rqctx.request.remote_addr().ip(); - nexus - .allow_list_upsert(&opctx, remote_addr, server_kind, params) - .await - .map(HttpResponseOk) - .map_err(HttpError::from) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn image_demote( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let image_lookup = nexus + .image_lookup( + &opctx, + params::ImageSelector { image: path.image, project: None }, + ) + .await?; -// Images - -/// List images -/// -/// List images which are global or scoped to the specified project. The images -/// are returned sorted by creation date, with the most recent images appearing first. -#[endpoint { - method = GET, - path = "/v1/images", - tags = ["images"], -}] -async fn image_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let parent_lookup = match scan_params.selector.project.clone() { - Some(project) => { - let project_lookup = nexus.project_lookup( + let project_lookup = nexus.project_lookup(&opctx, query)?; + + let image = nexus + .image_demote(&opctx, &image_lookup, &project_lookup) + .await?; + Ok(HttpResponseAccepted(image.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn instance_network_interface_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let instance_lookup = + nexus.instance_lookup(&opctx, scan_params.selector.clone())?; + let interfaces = nexus + .instance_network_interface_list( &opctx, - params::ProjectSelector { project }, - )?; - ImageParentLookup::Project(project_lookup) - } - None => { - let silo_lookup = nexus.current_silo_lookup(&opctx)?; - ImageParentLookup::Silo(silo_lookup) - } + &instance_lookup, + &paginated_by, + ) + .await? + .into_iter() + .map(|d| d.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + interfaces, + &marker_for_name_or_id, + )?)) }; - let images = nexus - .image_list(&opctx, &parent_lookup, &paginated_by) - .await? - .into_iter() - .map(|d| d.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - images, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create image -/// -/// Create a new image in a project. -#[endpoint { - method = POST, - path = "/v1/images", - tags = ["images"] -}] -async fn image_create( - rqctx: RequestContext, - query_params: Query, - new_image: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let params = &new_image.into_inner(); - let parent_lookup = match query.project.clone() { - Some(project) => { - let project_lookup = nexus.project_lookup( + async fn instance_network_interface_create( + rqctx: RequestContext, + query_params: Query, + interface_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let instance_lookup = nexus.instance_lookup(&opctx, query)?; + let iface = nexus + .network_interface_create( &opctx, - params::ProjectSelector { project }, - )?; - ImageParentLookup::Project(project_lookup) - } - None => { - let silo_lookup = nexus.current_silo_lookup(&opctx)?; - ImageParentLookup::Silo(silo_lookup) - } + &instance_lookup, + &interface_params.into_inner(), + ) + .await?; + Ok(HttpResponseCreated(iface.into())) }; - let image = nexus.image_create(&opctx, &parent_lookup, ¶ms).await?; - Ok(HttpResponseCreated(image.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch image -/// -/// Fetch the details for a specific image in a project. -#[endpoint { - method = GET, - path = "/v1/images/{image}", - tags = ["images"], -}] -async fn image_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let image: nexus_db_model::Image = match nexus - .image_lookup( + async fn instance_network_interface_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let interface_selector = params::InstanceNetworkInterfaceSelector { + project: query.project, + instance: query.instance, + network_interface: path.interface, + }; + let interface_lookup = nexus.instance_network_interface_lookup( &opctx, - params::ImageSelector { - image: path.image, - project: query.project, - }, - ) - .await? - { - ImageLookup::ProjectImage(image) => { - let (.., db_image) = image.fetch().await?; - db_image.into() - } - ImageLookup::SiloImage(image) => { - let (.., db_image) = image.fetch().await?; - db_image.into() - } + interface_selector, + )?; + nexus + .instance_network_interface_delete(&opctx, &interface_lookup) + .await?; + Ok(HttpResponseDeleted()) }; - Ok(HttpResponseOk(image.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete image -/// -/// Permanently delete an image from a project. This operation cannot be undone. -/// Any instances in the project using the image will continue to run, however -/// new instances can not be created with this image. -#[endpoint { - method = DELETE, - path = "/v1/images/{image}", - tags = ["images"], -}] -async fn image_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let image_lookup = nexus - .image_lookup( - &opctx, - params::ImageSelector { - image: path.image, - project: query.project, - }, - ) - .await?; - nexus.image_delete(&opctx, &image_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn instance_network_interface_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let interface_selector = params::InstanceNetworkInterfaceSelector { + project: query.project, + instance: query.instance, + network_interface: path.interface, + }; + let (.., interface) = nexus + .instance_network_interface_lookup(&opctx, interface_selector)? + .fetch() + .await?; + Ok(HttpResponseOk(interface.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Promote project image -/// -/// Promote project image to be visible to all projects in the silo -#[endpoint { - method = POST, - path = "/v1/images/{image}/promote", - tags = ["images"] -}] -async fn image_promote( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let image_lookup = nexus - .image_lookup( - &opctx, - params::ImageSelector { - image: path.image, + async fn instance_network_interface_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + updated_iface: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let updated_iface = updated_iface.into_inner(); + let network_interface_selector = + params::InstanceNetworkInterfaceSelector { project: query.project, - }, - ) - .await?; - let image = nexus.image_promote(&opctx, &image_lookup).await?; - Ok(HttpResponseAccepted(image.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + instance: query.instance, + network_interface: path.interface, + }; + let network_interface_lookup = nexus + .instance_network_interface_lookup( + &opctx, + network_interface_selector, + )?; + let interface = nexus + .instance_network_interface_update( + &opctx, + &network_interface_lookup, + updated_iface, + ) + .await?; + Ok(HttpResponseOk(InstanceNetworkInterface::from(interface))) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Demote silo image -/// -/// Demote silo image to be visible only to a specified project -#[endpoint { - method = POST, - path = "/v1/images/{image}/demote", - tags = ["images"] -}] -async fn image_demote( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let image_lookup = nexus - .image_lookup( - &opctx, - params::ImageSelector { image: path.image, project: None }, - ) - .await?; - - let project_lookup = nexus.project_lookup(&opctx, query)?; - - let image = - nexus.image_demote(&opctx, &image_lookup, &project_lookup).await?; - Ok(HttpResponseAccepted(image.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // External IP addresses for instances + + async fn instance_external_ip_list( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let ips = nexus + .instance_list_external_ips(&opctx, &instance_lookup) + .await?; + Ok(HttpResponseOk(ResultsPage { items: ips, next_page: None })) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List network interfaces -#[endpoint { - method = GET, - path = "/v1/network-interfaces", - tags = ["instances"], -}] -async fn instance_network_interface_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let instance_lookup = - nexus.instance_lookup(&opctx, scan_params.selector.clone())?; - let interfaces = nexus - .instance_network_interface_list( - &opctx, - &instance_lookup, - &paginated_by, - ) - .await? - .into_iter() - .map(|d| d.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - interfaces, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn instance_ephemeral_ip_attach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ip_to_create: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + let ip = nexus + .instance_attach_ephemeral_ip( + &opctx, + &instance_lookup, + ip_to_create.into_inner().pool, + ) + .await?; + Ok(HttpResponseAccepted(ip)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create network interface -#[endpoint { - method = POST, - path = "/v1/network-interfaces", - tags = ["instances"], -}] -async fn instance_network_interface_create( - rqctx: RequestContext, - query_params: Query, - interface_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let instance_lookup = nexus.instance_lookup(&opctx, query)?; - let iface = nexus - .network_interface_create( - &opctx, - &instance_lookup, - &interface_params.into_inner(), - ) - .await?; - Ok(HttpResponseCreated(iface.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn instance_ephemeral_ip_detach( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let instance_selector = params::InstanceSelector { + project: query.project, + instance: path.instance, + }; + let instance_lookup = + nexus.instance_lookup(&opctx, instance_selector)?; + nexus + .instance_detach_external_ip( + &opctx, + &instance_lookup, + ¶ms::ExternalIpDetach::Ephemeral, + ) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete network interface -/// -/// Note that the primary interface for an instance cannot be deleted if there -/// are any secondary interfaces. A new primary interface must be designated -/// first. The primary interface can be deleted if there are no secondary -/// interfaces. -#[endpoint { - method = DELETE, - path = "/v1/network-interfaces/{interface}", - tags = ["instances"], -}] -async fn instance_network_interface_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let interface_selector = params::InstanceNetworkInterfaceSelector { - project: query.project, - instance: query.instance, - network_interface: path.interface, - }; - let interface_lookup = nexus - .instance_network_interface_lookup(&opctx, interface_selector)?; - nexus - .instance_network_interface_delete(&opctx, &interface_lookup) - .await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Snapshots + + async fn snapshot_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let snapshots = nexus + .snapshot_list(&opctx, &project_lookup, &paginated_by) + .await? + .into_iter() + .map(|d| d.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + snapshots, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch network interface -#[endpoint { - method = GET, - path = "/v1/network-interfaces/{interface}", - tags = ["instances"], -}] -async fn instance_network_interface_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let interface_selector = params::InstanceNetworkInterfaceSelector { - project: query.project, - instance: query.instance, - network_interface: path.interface, - }; - let (.., interface) = nexus - .instance_network_interface_lookup(&opctx, interface_selector)? - .fetch() - .await?; - Ok(HttpResponseOk(interface.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn snapshot_create( + rqctx: RequestContext, + query_params: Query, + new_snapshot: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let new_snapshot_params = &new_snapshot.into_inner(); + let project_lookup = nexus.project_lookup(&opctx, query)?; + let snapshot = nexus + .snapshot_create(&opctx, project_lookup, &new_snapshot_params) + .await?; + Ok(HttpResponseCreated(snapshot.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update network interface -#[endpoint { - method = PUT, - path = "/v1/network-interfaces/{interface}", - tags = ["instances"], -}] -async fn instance_network_interface_update( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - updated_iface: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let updated_iface = updated_iface.into_inner(); - let network_interface_selector = - params::InstanceNetworkInterfaceSelector { + async fn snapshot_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let snapshot_selector = params::SnapshotSelector { project: query.project, - instance: query.instance, - network_interface: path.interface, + snapshot: path.snapshot, }; - let network_interface_lookup = nexus - .instance_network_interface_lookup( - &opctx, - network_interface_selector, - )?; - let interface = nexus - .instance_network_interface_update( - &opctx, - &network_interface_lookup, - updated_iface, - ) - .await?; - Ok(HttpResponseOk(InstanceNetworkInterface::from(interface))) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let (.., snapshot) = nexus + .snapshot_lookup(&opctx, snapshot_selector)? + .fetch() + .await?; + Ok(HttpResponseOk(snapshot.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// External IP addresses for instances - -/// List external IP addresses -#[endpoint { - method = GET, - path = "/v1/instances/{instance}/external-ips", - tags = ["instances"], -}] -async fn instance_external_ip_list( - rqctx: RequestContext, - query_params: Query, - path_params: Path, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let instance_selector = params::InstanceSelector { - project: query.project, - instance: path.instance, + async fn snapshot_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let snapshot_selector = params::SnapshotSelector { + project: query.project, + snapshot: path.snapshot, + }; + let snapshot_lookup = + nexus.snapshot_lookup(&opctx, snapshot_selector)?; + nexus.snapshot_delete(&opctx, &snapshot_lookup).await?; + Ok(HttpResponseDeleted()) }; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let ips = - nexus.instance_list_external_ips(&opctx, &instance_lookup).await?; - Ok(HttpResponseOk(ResultsPage { items: ips, next_page: None })) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Allocate and attach ephemeral IP to instance -#[endpoint { - method = POST, - path = "/v1/instances/{instance}/external-ips/ephemeral", - tags = ["instances"], -}] -async fn instance_ephemeral_ip_attach( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - ip_to_create: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let instance_selector = params::InstanceSelector { - project: query.project, - instance: path.instance, + // VPCs + + async fn vpc_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let vpcs = nexus + .vpc_list(&opctx, &project_lookup, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + vpcs, + &marker_for_name_or_id, + )?)) }; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - let ip = nexus - .instance_attach_ephemeral_ip( - &opctx, - &instance_lookup, - ip_to_create.into_inner().pool, - ) - .await?; - Ok(HttpResponseAccepted(ip)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Detach and deallocate ephemeral IP from instance -#[endpoint { - method = DELETE, - path = "/v1/instances/{instance}/external-ips/ephemeral", - tags = ["instances"], -}] -async fn instance_ephemeral_ip_detach( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + async fn vpc_create( + rqctx: RequestContext, + query_params: Query, + body: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); let query = query_params.into_inner(); - let instance_selector = params::InstanceSelector { - project: query.project, - instance: path.instance, + let new_vpc_params = body.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_lookup = nexus.project_lookup(&opctx, query)?; + let vpc = nexus + .project_create_vpc(&opctx, &project_lookup, &new_vpc_params) + .await?; + Ok(HttpResponseCreated(vpc.into())) }; - let instance_lookup = - nexus.instance_lookup(&opctx, instance_selector)?; - nexus - .instance_detach_external_ip( - &opctx, - &instance_lookup, - ¶ms::ExternalIpDetach::Ephemeral, - ) - .await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Snapshots - -/// List snapshots -#[endpoint { - method = GET, - path = "/v1/snapshots", - tags = ["snapshots"], -}] -async fn snapshot_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let project_lookup = - nexus.project_lookup(&opctx, scan_params.selector.clone())?; - let snapshots = nexus - .snapshot_list(&opctx, &project_lookup, &paginated_by) - .await? - .into_iter() - .map(|d| d.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - snapshots, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let vpc_selector = + params::VpcSelector { project: query.project, vpc: path.vpc }; + let (.., vpc) = + nexus.vpc_lookup(&opctx, vpc_selector)?.fetch().await?; + Ok(HttpResponseOk(vpc.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create snapshot -/// -/// Creates a point-in-time snapshot from a disk. -#[endpoint { - method = POST, - path = "/v1/snapshots", - tags = ["snapshots"], -}] -async fn snapshot_create( - rqctx: RequestContext, - query_params: Query, - new_snapshot: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let new_snapshot_params = &new_snapshot.into_inner(); - let project_lookup = nexus.project_lookup(&opctx, query)?; - let snapshot = nexus - .snapshot_create(&opctx, project_lookup, &new_snapshot_params) - .await?; - Ok(HttpResponseCreated(snapshot.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + updated_vpc: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let updated_vpc_params = &updated_vpc.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let vpc_selector = + params::VpcSelector { project: query.project, vpc: path.vpc }; + let vpc_lookup = nexus.vpc_lookup(&opctx, vpc_selector)?; + let vpc = nexus + .project_update_vpc(&opctx, &vpc_lookup, &updated_vpc_params) + .await?; + Ok(HttpResponseOk(vpc.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch snapshot -#[endpoint { - method = GET, - path = "/v1/snapshots/{snapshot}", - tags = ["snapshots"], -}] -async fn snapshot_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let snapshot_selector = params::SnapshotSelector { - project: query.project, - snapshot: path.snapshot, - }; - let (.., snapshot) = - nexus.snapshot_lookup(&opctx, snapshot_selector)?.fetch().await?; - Ok(HttpResponseOk(snapshot.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let vpc_selector = + params::VpcSelector { project: query.project, vpc: path.vpc }; + let vpc_lookup = nexus.vpc_lookup(&opctx, vpc_selector)?; + nexus.project_delete_vpc(&opctx, &vpc_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete snapshot -#[endpoint { - method = DELETE, - path = "/v1/snapshots/{snapshot}", - tags = ["snapshots"], -}] -async fn snapshot_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let snapshot_selector = params::SnapshotSelector { - project: query.project, - snapshot: path.snapshot, - }; - let snapshot_lookup = - nexus.snapshot_lookup(&opctx, snapshot_selector)?; - nexus.snapshot_delete(&opctx, &snapshot_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_subnet_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let vpc_lookup = + nexus.vpc_lookup(&opctx, scan_params.selector.clone())?; + let subnets = nexus + .vpc_subnet_list(&opctx, &vpc_lookup, &paginated_by) + .await? + .into_iter() + .map(|vpc| vpc.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + subnets, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// VPCs - -/// List VPCs -#[endpoint { - method = GET, - path = "/v1/vpcs", - tags = ["vpcs"], -}] -async fn vpc_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_lookup = - nexus.project_lookup(&opctx, scan_params.selector.clone())?; - let vpcs = nexus - .vpc_list(&opctx, &project_lookup, &paginated_by) - .await? - .into_iter() - .map(|p| p.into()) - .collect(); - - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - vpcs, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_subnet_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let create = create_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let vpc_lookup = nexus.vpc_lookup(&opctx, query)?; + let subnet = + nexus.vpc_create_subnet(&opctx, &vpc_lookup, &create).await?; + Ok(HttpResponseCreated(subnet.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create VPC -#[endpoint { - method = POST, - path = "/v1/vpcs", - tags = ["vpcs"], -}] -async fn vpc_create( - rqctx: RequestContext, - query_params: Query, - body: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let new_vpc_params = body.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_lookup = nexus.project_lookup(&opctx, query)?; - let vpc = nexus - .project_create_vpc(&opctx, &project_lookup, &new_vpc_params) - .await?; - Ok(HttpResponseCreated(vpc.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_subnet_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let subnet_selector = params::SubnetSelector { + project: query.project, + vpc: query.vpc, + subnet: path.subnet, + }; + let (.., subnet) = nexus + .vpc_subnet_lookup(&opctx, subnet_selector)? + .fetch() + .await?; + Ok(HttpResponseOk(subnet.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch VPC -#[endpoint { - method = GET, - path = "/v1/vpcs/{vpc}", - tags = ["vpcs"], -}] -async fn vpc_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let vpc_selector = - params::VpcSelector { project: query.project, vpc: path.vpc }; - let (.., vpc) = nexus.vpc_lookup(&opctx, vpc_selector)?.fetch().await?; - Ok(HttpResponseOk(vpc.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_subnet_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let subnet_selector = params::SubnetSelector { + project: query.project, + vpc: query.vpc, + subnet: path.subnet, + }; + let subnet_lookup = + nexus.vpc_subnet_lookup(&opctx, subnet_selector)?; + nexus.vpc_delete_subnet(&opctx, &subnet_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update a VPC -#[endpoint { - method = PUT, - path = "/v1/vpcs/{vpc}", - tags = ["vpcs"], -}] -async fn vpc_update( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - updated_vpc: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let updated_vpc_params = &updated_vpc.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let vpc_selector = - params::VpcSelector { project: query.project, vpc: path.vpc }; - let vpc_lookup = nexus.vpc_lookup(&opctx, vpc_selector)?; - let vpc = nexus - .project_update_vpc(&opctx, &vpc_lookup, &updated_vpc_params) - .await?; - Ok(HttpResponseOk(vpc.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_subnet_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + subnet_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let subnet_params = subnet_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let subnet_selector = params::SubnetSelector { + project: query.project, + vpc: query.vpc, + subnet: path.subnet, + }; + let subnet_lookup = + nexus.vpc_subnet_lookup(&opctx, subnet_selector)?; + let subnet = nexus + .vpc_update_subnet(&opctx, &subnet_lookup, &subnet_params) + .await?; + Ok(HttpResponseOk(subnet.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete VPC -#[endpoint { - method = DELETE, - path = "/v1/vpcs/{vpc}", - tags = ["vpcs"], -}] -async fn vpc_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let vpc_selector = - params::VpcSelector { project: query.project, vpc: path.vpc }; - let vpc_lookup = nexus.vpc_lookup(&opctx, vpc_selector)?; - nexus.project_delete_vpc(&opctx, &vpc_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // This endpoint is likely temporary. We would rather list all IPs allocated in + // a subnet whether they come from NICs or something else. See + // https://github.com/oxidecomputer/omicron/issues/2476 -/// List subnets -#[endpoint { - method = GET, - path = "/v1/vpc-subnets", - tags = ["vpcs"], -}] -async fn vpc_subnet_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let vpc_lookup = - nexus.vpc_lookup(&opctx, scan_params.selector.clone())?; - let subnets = nexus - .vpc_subnet_list(&opctx, &vpc_lookup, &paginated_by) - .await? - .into_iter() - .map(|vpc| vpc.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - subnets, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_subnet_list_network_interfaces( + rqctx: RequestContext, + path_params: Path, + query_params: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let path = path_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let subnet_selector = params::SubnetSelector { + project: scan_params.selector.project.clone(), + vpc: scan_params.selector.vpc.clone(), + subnet: path.subnet, + }; + let subnet_lookup = + nexus.vpc_subnet_lookup(&opctx, subnet_selector)?; + let interfaces = nexus + .subnet_list_instance_network_interfaces( + &opctx, + &subnet_lookup, + &paginated_by, + ) + .await? + .into_iter() + .map(|interfaces| interfaces.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + interfaces, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create subnet -#[endpoint { - method = POST, - path = "/v1/vpc-subnets", - tags = ["vpcs"], -}] -async fn vpc_subnet_create( - rqctx: RequestContext, - query_params: Query, - create_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let create = create_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let vpc_lookup = nexus.vpc_lookup(&opctx, query)?; - let subnet = - nexus.vpc_create_subnet(&opctx, &vpc_lookup, &create).await?; - Ok(HttpResponseCreated(subnet.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // VPC Firewalls + + async fn vpc_firewall_rules_view( + rqctx: RequestContext, + query_params: Query, + ) -> Result, HttpError> { + // TODO: Check If-Match and fail if the ETag doesn't match anymore. + // Without this check, if firewall rules change while someone is listing + // the rules, they will see a mix of the old and new rules. + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let vpc_lookup = nexus.vpc_lookup(&opctx, query)?; + let rules = + nexus.vpc_list_firewall_rules(&opctx, &vpc_lookup).await?; + Ok(HttpResponseOk(VpcFirewallRules { + rules: rules.into_iter().map(|rule| rule.into()).collect(), + })) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch subnet -#[endpoint { - method = GET, - path = "/v1/vpc-subnets/{subnet}", - tags = ["vpcs"], -}] -async fn vpc_subnet_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let subnet_selector = params::SubnetSelector { - project: query.project, - vpc: query.vpc, - subnet: path.subnet, - }; - let (.., subnet) = - nexus.vpc_subnet_lookup(&opctx, subnet_selector)?.fetch().await?; - Ok(HttpResponseOk(subnet.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Note: the limits in the below comment come from the firewall rules model + // file, nexus/db-model/src/vpc_firewall_rule.rs. + + async fn vpc_firewall_rules_update( + rqctx: RequestContext, + query_params: Query, + router_params: TypedBody, + ) -> Result, HttpError> { + // TODO: Check If-Match and fail if the ETag doesn't match anymore. + // TODO: limit size of the ruleset because the GET endpoint is not paginated + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let router_params = router_params.into_inner(); + let vpc_lookup = nexus.vpc_lookup(&opctx, query)?; + let rules = nexus + .vpc_update_firewall_rules(&opctx, &vpc_lookup, &router_params) + .await?; + Ok(HttpResponseOk(VpcFirewallRules { + rules: rules.into_iter().map(|rule| rule.into()).collect(), + })) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete subnet -#[endpoint { - method = DELETE, - path = "/v1/vpc-subnets/{subnet}", - tags = ["vpcs"], -}] -async fn vpc_subnet_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let subnet_selector = params::SubnetSelector { - project: query.project, - vpc: query.vpc, - subnet: path.subnet, - }; - let subnet_lookup = nexus.vpc_subnet_lookup(&opctx, subnet_selector)?; - nexus.vpc_delete_subnet(&opctx, &subnet_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // VPC Routers + + async fn vpc_router_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let vpc_lookup = + nexus.vpc_lookup(&opctx, scan_params.selector.clone())?; + let routers = nexus + .vpc_router_list(&opctx, &vpc_lookup, &paginated_by) + .await? + .into_iter() + .map(|s| s.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + routers, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update subnet -#[endpoint { - method = PUT, - path = "/v1/vpc-subnets/{subnet}", - tags = ["vpcs"], -}] -async fn vpc_subnet_update( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - subnet_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let subnet_params = subnet_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let subnet_selector = params::SubnetSelector { - project: query.project, - vpc: query.vpc, - subnet: path.subnet, - }; - let subnet_lookup = nexus.vpc_subnet_lookup(&opctx, subnet_selector)?; - let subnet = nexus - .vpc_update_subnet(&opctx, &subnet_lookup, &subnet_params) - .await?; - Ok(HttpResponseOk(subnet.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_router_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let router_selector = params::RouterSelector { + project: query.project, + vpc: query.vpc, + router: path.router, + }; + let (.., vpc_router) = nexus + .vpc_router_lookup(&opctx, router_selector)? + .fetch() + .await?; + Ok(HttpResponseOk(vpc_router.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// This endpoint is likely temporary. We would rather list all IPs allocated in -// a subnet whether they come from NICs or something else. See -// https://github.com/oxidecomputer/omicron/issues/2476 - -/// List network interfaces -#[endpoint { - method = GET, - path = "/v1/vpc-subnets/{subnet}/network-interfaces", - tags = ["vpcs"], -}] -async fn vpc_subnet_list_network_interfaces( - rqctx: RequestContext, - path_params: Path, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let path = path_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let subnet_selector = params::SubnetSelector { - project: scan_params.selector.project.clone(), - vpc: scan_params.selector.vpc.clone(), - subnet: path.subnet, - }; - let subnet_lookup = nexus.vpc_subnet_lookup(&opctx, subnet_selector)?; - let interfaces = nexus - .subnet_list_instance_network_interfaces( - &opctx, - &subnet_lookup, - &paginated_by, - ) - .await? - .into_iter() - .map(|interfaces| interfaces.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - interfaces, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_router_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let create = create_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let vpc_lookup = nexus.vpc_lookup(&opctx, query)?; + let router = nexus + .vpc_create_router( + &opctx, + &vpc_lookup, + &db::model::VpcRouterKind::Custom, + &create, + ) + .await?; + Ok(HttpResponseCreated(router.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// VPC Firewalls - -/// List firewall rules -#[endpoint { - method = GET, - path = "/v1/vpc-firewall-rules", - tags = ["vpcs"], -}] -async fn vpc_firewall_rules_view( - rqctx: RequestContext, - query_params: Query, -) -> Result, HttpError> { - // TODO: Check If-Match and fail if the ETag doesn't match anymore. - // Without this check, if firewall rules change while someone is listing - // the rules, they will see a mix of the old and new rules. - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let vpc_lookup = nexus.vpc_lookup(&opctx, query)?; - let rules = nexus.vpc_list_firewall_rules(&opctx, &vpc_lookup).await?; - Ok(HttpResponseOk(VpcFirewallRules { - rules: rules.into_iter().map(|rule| rule.into()).collect(), - })) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_router_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let router_selector = params::RouterSelector { + project: query.project, + vpc: query.vpc, + router: path.router, + }; + let router_lookup = + nexus.vpc_router_lookup(&opctx, router_selector)?; + nexus.vpc_delete_router(&opctx, &router_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Note: the limits in the below comment come from the firewall rules model -// file, nexus/db-model/src/vpc_firewall_rule.rs. - -/// Replace firewall rules -/// -/// The maximum number of rules per VPC is 1024. -/// -/// Targets are used to specify the set of instances to which a firewall rule -/// applies. You can target instances directly by name, or specify a VPC, VPC -/// subnet, IP, or IP subnet, which will apply the rule to traffic going to -/// all matching instances. Targets are additive: the rule applies to instances -/// matching ANY target. The maximum number of targets is 256. -/// -/// Filters reduce the scope of a firewall rule. Without filters, the rule -/// applies to all packets to the targets (or from the targets, if it's an -/// outbound rule). With multiple filters, the rule applies only to packets -/// matching ALL filters. The maximum number of each type of filter is 256. -#[endpoint { - method = PUT, - path = "/v1/vpc-firewall-rules", - tags = ["vpcs"], -}] -async fn vpc_firewall_rules_update( - rqctx: RequestContext, - query_params: Query, - router_params: TypedBody, -) -> Result, HttpError> { - // TODO: Check If-Match and fail if the ETag doesn't match anymore. - // TODO: limit size of the ruleset because the GET endpoint is not paginated - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let router_params = router_params.into_inner(); - let vpc_lookup = nexus.vpc_lookup(&opctx, query)?; - let rules = nexus - .vpc_update_firewall_rules(&opctx, &vpc_lookup, &router_params) - .await?; - Ok(HttpResponseOk(VpcFirewallRules { - rules: rules.into_iter().map(|rule| rule.into()).collect(), - })) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_router_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + router_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let router_params = router_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let router_selector = params::RouterSelector { + project: query.project, + vpc: query.vpc, + router: path.router, + }; + let router_lookup = + nexus.vpc_router_lookup(&opctx, router_selector)?; + let router = nexus + .vpc_update_router(&opctx, &router_lookup, &router_params) + .await?; + Ok(HttpResponseOk(router.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// VPC Routers - -/// List routers -#[endpoint { - method = GET, - path = "/v1/vpc-routers", - tags = ["vpcs"], -}] -async fn vpc_router_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let vpc_lookup = - nexus.vpc_lookup(&opctx, scan_params.selector.clone())?; - let routers = nexus - .vpc_router_list(&opctx, &vpc_lookup, &paginated_by) - .await? - .into_iter() - .map(|s| s.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - routers, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_router_route_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let router_lookup = nexus + .vpc_router_lookup(&opctx, scan_params.selector.clone())?; + let routes = nexus + .vpc_router_route_list(&opctx, &router_lookup, &paginated_by) + .await? + .into_iter() + .map(|route| route.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + routes, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch router -#[endpoint { - method = GET, - path = "/v1/vpc-routers/{router}", - tags = ["vpcs"], -}] -async fn vpc_router_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let router_selector = params::RouterSelector { - project: query.project, - vpc: query.vpc, - router: path.router, - }; - let (.., vpc_router) = - nexus.vpc_router_lookup(&opctx, router_selector)?.fetch().await?; - Ok(HttpResponseOk(vpc_router.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Vpc Router Routes + + async fn vpc_router_route_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let route_selector = params::RouteSelector { + project: query.project, + vpc: query.vpc, + router: Some(query.router), + route: path.route, + }; + let (.., route) = nexus + .vpc_router_route_lookup(&opctx, route_selector)? + .fetch() + .await?; + Ok(HttpResponseOk(route.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create VPC router -#[endpoint { - method = POST, - path = "/v1/vpc-routers", - tags = ["vpcs"], -}] -async fn vpc_router_create( - rqctx: RequestContext, - query_params: Query, - create_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let create = create_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let vpc_lookup = nexus.vpc_lookup(&opctx, query)?; - let router = nexus - .vpc_create_router( - &opctx, - &vpc_lookup, - &db::model::VpcRouterKind::Custom, - &create, - ) - .await?; - Ok(HttpResponseCreated(router.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_router_route_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let create = create_params.into_inner(); + let router_lookup = nexus.vpc_router_lookup(&opctx, query)?; + let route = nexus + .router_create_route( + &opctx, + &router_lookup, + &RouterRouteKind::Custom, + &create, + ) + .await?; + Ok(HttpResponseCreated(route.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete router -#[endpoint { - method = DELETE, - path = "/v1/vpc-routers/{router}", - tags = ["vpcs"], -}] -async fn vpc_router_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let router_selector = params::RouterSelector { - project: query.project, - vpc: query.vpc, - router: path.router, - }; - let router_lookup = nexus.vpc_router_lookup(&opctx, router_selector)?; - nexus.vpc_delete_router(&opctx, &router_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_router_route_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let route_selector = params::RouteSelector { + project: query.project, + vpc: query.vpc, + router: query.router, + route: path.route, + }; + let route_lookup = + nexus.vpc_router_route_lookup(&opctx, route_selector)?; + nexus.router_delete_route(&opctx, &route_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update router -#[endpoint { - method = PUT, - path = "/v1/vpc-routers/{router}", - tags = ["vpcs"], -}] -async fn vpc_router_update( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - router_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let router_params = router_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let router_selector = params::RouterSelector { - project: query.project, - vpc: query.vpc, - router: path.router, - }; - let router_lookup = nexus.vpc_router_lookup(&opctx, router_selector)?; - let router = nexus - .vpc_update_router(&opctx, &router_lookup, &router_params) - .await?; - Ok(HttpResponseOk(router.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn vpc_router_route_update( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + router_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let router_params = router_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let route_selector = params::RouteSelector { + project: query.project, + vpc: query.vpc, + router: query.router, + route: path.route, + }; + let route_lookup = + nexus.vpc_router_route_lookup(&opctx, route_selector)?; + let route = nexus + .router_update_route(&opctx, &route_lookup, &router_params) + .await?; + Ok(HttpResponseOk(route.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List routes -/// -/// List the routes associated with a router in a particular VPC. -#[endpoint { - method = GET, - path = "/v1/vpc-router-routes", - tags = ["vpcs"], -}] -async fn vpc_router_route_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let router_lookup = - nexus.vpc_router_lookup(&opctx, scan_params.selector.clone())?; - let routes = nexus - .vpc_router_route_list(&opctx, &router_lookup, &paginated_by) - .await? - .into_iter() - .map(|route| route.into()) - .collect(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - routes, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Racks + + async fn rack_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let racks = nexus + .racks_list(&opctx, &data_page_params_for(&rqctx, &query)?) + .await? + .into_iter() + .map(|r| r.into()) + .collect(); + Ok(HttpResponseOk(ScanById::results_page( + &query, + racks, + &|_, rack: &Rack| rack.identity.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Vpc Router Routes - -/// Fetch route -#[endpoint { - method = GET, - path = "/v1/vpc-router-routes/{route}", - tags = ["vpcs"], -}] -async fn vpc_router_route_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let route_selector = params::RouteSelector { - project: query.project, - vpc: query.vpc, - router: Some(query.router), - route: path.route, - }; - let (.., route) = nexus - .vpc_router_route_lookup(&opctx, route_selector)? - .fetch() - .await?; - Ok(HttpResponseOk(route.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn rack_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let rack_info = nexus.rack_lookup(&opctx, &path.rack_id).await?; + Ok(HttpResponseOk(rack_info.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Create route -#[endpoint { - method = POST, - path = "/v1/vpc-router-routes", - tags = ["vpcs"], -}] -async fn vpc_router_route_create( - rqctx: RequestContext, - query_params: Query, - create_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let create = create_params.into_inner(); - let router_lookup = nexus.vpc_router_lookup(&opctx, query)?; - let route = nexus - .router_create_route( - &opctx, - &router_lookup, - &RouterRouteKind::Custom, - &create, + async fn sled_list_uninitialized( + rqctx: RequestContext, + query: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + // We don't actually support real pagination + let pag_params = query.into_inner(); + if let dropshot::WhichPage::Next(last_seen) = &pag_params.page { + return Err(Error::invalid_value( + last_seen.clone(), + "bad page token", ) - .await?; - Ok(HttpResponseCreated(route.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + .into()); + } + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let sleds = nexus.sled_list_uninitialized(&opctx).await?; + Ok(HttpResponseOk(ResultsPage { items: sleds, next_page: None })) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Delete route -#[endpoint { - method = DELETE, - path = "/v1/vpc-router-routes/{route}", - tags = ["vpcs"], -}] -async fn vpc_router_route_delete( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let route_selector = params::RouteSelector { - project: query.project, - vpc: query.vpc, - router: query.router, - route: path.route, - }; - let route_lookup = - nexus.vpc_router_route_lookup(&opctx, route_selector)?; - nexus.router_delete_route(&opctx, &route_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn sled_add( + rqctx: RequestContext, + sled: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let nexus = &apictx.context.nexus; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let id = nexus + .sled_add(&opctx, sled.into_inner()) + .await? + .into_untyped_uuid(); + Ok(HttpResponseCreated(views::SledId { id })) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Update route -#[endpoint { - method = PUT, - path = "/v1/vpc-router-routes/{route}", - tags = ["vpcs"], -}] -async fn vpc_router_route_update( - rqctx: RequestContext, - path_params: Path, - query_params: Query, - router_params: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let router_params = router_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let route_selector = params::RouteSelector { - project: query.project, - vpc: query.vpc, - router: query.router, - route: path.route, - }; - let route_lookup = - nexus.vpc_router_route_lookup(&opctx, route_selector)?; - let route = nexus - .router_update_route(&opctx, &route_lookup, &router_params) - .await?; - Ok(HttpResponseOk(route.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Sleds + + async fn sled_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let sleds = nexus + .sled_list(&opctx, &data_page_params_for(&rqctx, &query)?) + .await? + .into_iter() + .map(|s| s.into()) + .collect(); + Ok(HttpResponseOk(ScanById::results_page( + &query, + sleds, + &|_, sled: &Sled| sled.identity.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Racks - -/// List racks -#[endpoint { - method = GET, - path = "/v1/system/hardware/racks", - tags = ["system/hardware"], -}] -async fn rack_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let racks = nexus - .racks_list(&opctx, &data_page_params_for(&rqctx, &query)?) - .await? - .into_iter() - .map(|r| r.into()) - .collect(); - Ok(HttpResponseOk(ScanById::results_page( - &query, - racks, - &|_, rack: &Rack| rack.identity.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn sled_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let (.., sled) = + nexus.sled_lookup(&opctx, &path.sled_id)?.fetch().await?; + Ok(HttpResponseOk(sled.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Path parameters for Rack requests -#[derive(Deserialize, JsonSchema)] -struct RackPathParam { - /// The rack's unique ID. - rack_id: Uuid, -} + async fn sled_set_provision_policy( + rqctx: RequestContext, + path_params: Path, + new_provision_state: TypedBody, + ) -> Result, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; -/// Fetch rack -#[endpoint { - method = GET, - path = "/v1/system/hardware/racks/{rack_id}", - tags = ["system/hardware"], -}] -async fn rack_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let rack_info = nexus.rack_lookup(&opctx, &path.rack_id).await?; - Ok(HttpResponseOk(rack_info.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let path = path_params.into_inner(); + let new_state = new_provision_state.into_inner().state; -/// List uninitialized sleds -#[endpoint { - method = GET, - path = "/v1/system/hardware/sleds-uninitialized", - tags = ["system/hardware"] -}] -async fn sled_list_uninitialized( - rqctx: RequestContext, - query: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - // We don't actually support real pagination - let pag_params = query.into_inner(); - if let dropshot::WhichPage::Next(last_seen) = &pag_params.page { - return Err( - Error::invalid_value(last_seen.clone(), "bad page token").into() - ); - } - let handler = async { - let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let sleds = nexus.sled_list_uninitialized(&opctx).await?; - Ok(HttpResponseOk(ResultsPage { items: sleds, next_page: None })) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; -/// The unique ID of a sled. -#[derive(Clone, Debug, Serialize, JsonSchema)] -pub struct SledId { - pub id: Uuid, -} + let sled_lookup = nexus.sled_lookup(&opctx, &path.sled_id)?; -/// Add sled to initialized rack -// -// TODO: In the future this should really be a PUT request, once we resolve -// https://github.com/oxidecomputer/omicron/issues/4494. It should also -// explicitly be tied to a rack via a `rack_id` path param. For now we assume -// we are only operating on single rack systems. -#[endpoint { - method = POST, - path = "/v1/system/hardware/sleds", - tags = ["system/hardware"] -}] -async fn sled_add( - rqctx: RequestContext, - sled: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let id = nexus - .sled_add(&opctx, sled.into_inner()) - .await? - .into_untyped_uuid(); - Ok(HttpResponseCreated(SledId { id })) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let old_state = nexus + .sled_set_provision_policy(&opctx, &sled_lookup, new_state) + .await?; -// Sleds - -/// List sleds -#[endpoint { - method = GET, - path = "/v1/system/hardware/sleds", - tags = ["system/hardware"], -}] -async fn sled_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let sleds = nexus - .sled_list(&opctx, &data_page_params_for(&rqctx, &query)?) - .await? - .into_iter() - .map(|s| s.into()) - .collect(); - Ok(HttpResponseOk(ScanById::results_page( - &query, - sleds, - &|_, sled: &Sled| sled.identity.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let response = + params::SledProvisionPolicyResponse { old_state, new_state }; -/// Fetch sled -#[endpoint { - method = GET, - path = "/v1/system/hardware/sleds/{sled_id}", - tags = ["system/hardware"], -}] -async fn sled_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let (.., sled) = - nexus.sled_lookup(&opctx, &path.sled_id)?.fetch().await?; - Ok(HttpResponseOk(sled.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + Ok(HttpResponseOk(response)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Set sled provision policy -#[endpoint { - method = PUT, - path = "/v1/system/hardware/sleds/{sled_id}/provision-policy", - tags = ["system/hardware"], -}] -async fn sled_set_provision_policy( - rqctx: RequestContext, - path_params: Path, - new_provision_state: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; + async fn sled_instance_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let sled_lookup = nexus.sled_lookup(&opctx, &path.sled_id)?; + let sled_instances = nexus + .sled_instance_list( + &opctx, + &sled_lookup, + &data_page_params_for(&rqctx, &query)?, + ) + .await? + .into_iter() + .map(|s| s.into()) + .collect(); + Ok(HttpResponseOk(ScanById::results_page( + &query, + sled_instances, + &|_, sled_instance: &views::SledInstance| { + sled_instance.identity.id + }, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let path = path_params.into_inner(); - let new_state = new_provision_state.into_inner().state; + // Physical disks + + async fn physical_disk_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let disks = nexus + .physical_disk_list( + &opctx, + &data_page_params_for(&rqctx, &query)?, + ) + .await? + .into_iter() + .map(|s| s.into()) + .collect(); + Ok(HttpResponseOk(ScanById::results_page( + &query, + disks, + &|_, disk: &PhysicalDisk| disk.identity.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn physical_disk_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + + let (.., physical_disk) = + nexus.physical_disk_lookup(&opctx, &path)?.fetch().await?; + Ok(HttpResponseOk(physical_disk.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + // Switches + + async fn switch_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let switches = nexus + .switch_list(&opctx, &data_page_params_for(&rqctx, &query)?) + .await? + .into_iter() + .map(|s| s.into()) + .collect(); + Ok(HttpResponseOk(ScanById::results_page( + &query, + switches, + &|_, switch: &views::Switch| switch.identity.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn switch_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let (.., switch) = nexus + .switch_lookup( + &opctx, + params::SwitchSelector { switch: path.switch_id }, + )? + .fetch() + .await?; + Ok(HttpResponseOk(switch.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn sled_physical_disk_list( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let disks = nexus + .sled_list_physical_disks( + &opctx, + path.sled_id, + &data_page_params_for(&rqctx, &query)?, + ) + .await? + .into_iter() + .map(|s| s.into()) + .collect(); + Ok(HttpResponseOk(ScanById::results_page( + &query, + disks, + &|_, disk: &PhysicalDisk| disk.identity.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + // Metrics - let sled_lookup = nexus.sled_lookup(&opctx, &path.sled_id)?; + async fn system_metric( + rqctx: RequestContext, + path_params: Path, + pag_params: Query< + PaginationParams, + >, + other_params: Query, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let metric_name = path_params.into_inner().metric_name; + let pagination = pag_params.into_inner(); + let limit = rqctx.page_limit(&pagination)?; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let silo_lookup = match other_params.into_inner().silo { + Some(silo) => Some(nexus.silo_lookup(&opctx, silo)?), + _ => None, + }; - let old_state = nexus - .sled_set_provision_policy(&opctx, &sled_lookup, new_state) - .await?; + let result = nexus + .system_metric_list( + &opctx, + metric_name, + silo_lookup, + pagination, + limit, + ) + .await?; - let response = - params::SledProvisionPolicyResponse { old_state, new_state }; + Ok(HttpResponseOk(result)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - Ok(HttpResponseOk(response)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn silo_metric( + rqctx: RequestContext, + path_params: Path, + pag_params: Query< + PaginationParams, + >, + other_params: Query, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let metric_name = path_params.into_inner().metric_name; + + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let project_lookup = match other_params.into_inner().project { + Some(project) => { + let project_selector = params::ProjectSelector { project }; + Some(nexus.project_lookup(&opctx, project_selector)?) + } + _ => None, + }; -/// List instances running on given sled -#[endpoint { - method = GET, - path = "/v1/system/hardware/sleds/{sled_id}/instances", - tags = ["system/hardware"], -}] -async fn sled_instance_list( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let sled_lookup = nexus.sled_lookup(&opctx, &path.sled_id)?; - let sled_instances = nexus - .sled_instance_list( - &opctx, - &sled_lookup, - &data_page_params_for(&rqctx, &query)?, - ) - .await? - .into_iter() - .map(|s| s.into()) - .collect(); - Ok(HttpResponseOk(ScanById::results_page( - &query, - sled_instances, - &|_, sled_instance: &views::SledInstance| sled_instance.identity.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let pagination = pag_params.into_inner(); + let limit = rqctx.page_limit(&pagination)?; -// Physical disks - -/// List physical disks -#[endpoint { - method = GET, - path = "/v1/system/hardware/disks", - tags = ["system/hardware"], -}] -async fn physical_disk_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let disks = nexus - .physical_disk_list(&opctx, &data_page_params_for(&rqctx, &query)?) - .await? - .into_iter() - .map(|s| s.into()) - .collect(); - Ok(HttpResponseOk(ScanById::results_page( - &query, - disks, - &|_, disk: &PhysicalDisk| disk.identity.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus + .silo_metric_list( + &opctx, + metric_name, + project_lookup, + pagination, + limit, + ) + .await?; -/// Get a physical disk -#[endpoint { - method = GET, - path = "/v1/system/hardware/disks/{disk_id}", - tags = ["system/hardware"], -}] -async fn physical_disk_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + Ok(HttpResponseOk(result)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let (.., physical_disk) = - nexus.physical_disk_lookup(&opctx, &path)?.fetch().await?; - Ok(HttpResponseOk(physical_disk.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn timeseries_schema_list( + rqctx: RequestContext, + pag_params: Query, + ) -> Result< + HttpResponseOk>, + HttpError, + > { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let pagination = pag_params.into_inner(); + let limit = rqctx.page_limit(&pagination)?; + nexus + .timeseries_schema_list(&opctx, &pagination, limit) + .await + .map(HttpResponseOk) + .map_err(HttpError::from) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Switches - -/// List switches -#[endpoint { - method = GET, - path = "/v1/system/hardware/switches", - tags = ["system/hardware"], -}] -async fn switch_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let switches = nexus - .switch_list(&opctx, &data_page_params_for(&rqctx, &query)?) - .await? - .into_iter() - .map(|s| s.into()) - .collect(); - Ok(HttpResponseOk(ScanById::results_page( - &query, - switches, - &|_, switch: &views::Switch| switch.identity.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn timeseries_query( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = body.into_inner().query; + nexus + .timeseries_query(&opctx, &query) + .await + .map(|tables| HttpResponseOk(views::OxqlQueryResult { tables })) + .map_err(HttpError::from) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch switch -#[endpoint { - method = GET, - path = "/v1/system/hardware/switches/{switch_id}", - tags = ["system/hardware"], - }] -async fn switch_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let (.., switch) = nexus - .switch_lookup( - &opctx, - params::SwitchSelector { switch: path.switch_id }, - )? - .fetch() - .await?; - Ok(HttpResponseOk(switch.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Updates + + async fn system_update_put_repository( + rqctx: RequestContext, + query: Query, + body: StreamingBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let nexus = &apictx.context.nexus; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query.into_inner(); + let body = body.into_stream(); + let update = nexus + .updates_put_repository(&opctx, body, query.file_name) + .await?; + Ok(HttpResponseOk(update)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// List physical disks attached to sleds -#[endpoint { - method = GET, - path = "/v1/system/hardware/sleds/{sled_id}/disks", - tags = ["system/hardware"], -}] -async fn sled_physical_disk_list( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let query = query_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let disks = nexus - .sled_list_physical_disks( - &opctx, - path.sled_id, - &data_page_params_for(&rqctx, &query)?, - ) - .await? - .into_iter() - .map(|s| s.into()) - .collect(); - Ok(HttpResponseOk(ScanById::results_page( - &query, - disks, - &|_, disk: &PhysicalDisk| disk.identity.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn system_update_get_repository( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let nexus = &apictx.context.nexus; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let params = path_params.into_inner(); + let description = nexus + .updates_get_repository(&opctx, params.system_version) + .await?; + Ok(HttpResponseOk(TufRepoGetResponse { + description: description.into_external(), + })) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Metrics + // Silo users + + async fn user_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)?; + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let scan_params = ScanById::from_query(&query)?; + + // TODO: a valid UUID gets parsed here and will 404 if it doesn't exist + // (as expected) but a non-UUID string just gets let through as None + // (i.e., ignored) instead of 400ing + + let users = if let Some(group_id) = scan_params.selector.group { + nexus + .current_silo_group_users_list( + &opctx, &pagparams, &group_id, + ) + .await? + } else { + nexus.silo_users_list_current(&opctx, &pagparams).await? + }; -#[derive(Display, Deserialize, JsonSchema)] -#[display(style = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum SystemMetricName { - VirtualDiskSpaceProvisioned, - CpusProvisioned, - RamProvisioned, -} + Ok(HttpResponseOk(ScanById::results_page( + &query, + users.into_iter().map(|i| i.into()).collect(), + &|_, user: &User| user.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -#[derive(Deserialize, JsonSchema)] -struct SystemMetricsPathParam { - metric_name: SystemMetricName, -} + // Silo groups -/// View metrics -/// -/// View CPU, memory, or storage utilization metrics at the fleet or silo level. -#[endpoint { - method = GET, - path = "/v1/system/metrics/{metric_name}", - tags = ["system/metrics"], -}] -async fn system_metric( - rqctx: RequestContext, - path_params: Path, - pag_params: Query< - PaginationParams, - >, - other_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { + async fn group_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; - let metric_name = path_params.into_inner().metric_name; - let pagination = pag_params.into_inner(); - let limit = rqctx.page_limit(&pagination)?; + let query = query_params.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)?; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let groups = nexus + .silo_groups_list(&opctx, &pagparams) + .await? + .into_iter() + .map(|i| i.into()) + .collect(); + Ok(HttpResponseOk(ScanById::results_page( + &query, + groups, + &|_, group: &Group| group.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let silo_lookup = match other_params.into_inner().silo { - Some(silo) => Some(nexus.silo_lookup(&opctx, silo)?), - _ => None, + async fn group_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let (.., group) = + nexus.silo_group_lookup(&opctx, &path.group_id).fetch().await?; + Ok(HttpResponseOk(group.into())) }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let result = nexus - .system_metric_list( - &opctx, - metric_name, - silo_lookup, - pagination, - limit, - ) - .await?; - - Ok(HttpResponseOk(result)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Built-in (system) users -/// View metrics -/// -/// View CPU, memory, or storage utilization metrics at the silo or project level. -#[endpoint { - method = GET, - path = "/v1/metrics/{metric_name}", - tags = ["metrics"], -}] -async fn silo_metric( - rqctx: RequestContext, - path_params: Path, - pag_params: Query< - PaginationParams, - >, - other_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { + async fn user_builtin_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; - let metric_name = path_params.into_inner().metric_name; - - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let project_lookup = match other_params.into_inner().project { - Some(project) => { - let project_selector = params::ProjectSelector { project }; - Some(nexus.project_lookup(&opctx, project_selector)?) - } - _ => None, + let query = query_params.into_inner(); + let pagparams = data_page_params_for(&rqctx, &query)? + .map_name(|n| Name::ref_cast(n)); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let users = nexus + .users_builtin_list(&opctx, &pagparams) + .await? + .into_iter() + .map(|i| i.into()) + .collect(); + Ok(HttpResponseOk(ScanByName::results_page( + &query, + users, + &marker_for_name, + )?)) }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let pagination = pag_params.into_inner(); - let limit = rqctx.page_limit(&pagination)?; + async fn user_builtin_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let user_selector = path_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let (.., user) = nexus + .user_builtin_lookup(&opctx, &user_selector)? + .fetch() + .await?; + Ok(HttpResponseOk(user.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus - .silo_metric_list( - &opctx, - metric_name, - project_lookup, - pagination, - limit, - ) - .await?; - - Ok(HttpResponseOk(result)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Built-in roles -/// List timeseries schemas -#[endpoint { - method = GET, - path = "/v1/timeseries/schema", - tags = ["metrics"], -}] -async fn timeseries_schema_list( - rqctx: RequestContext, - pag_params: Query, -) -> Result>, HttpError> -{ - let apictx = rqctx.context(); - let handler = async { + async fn role_list( + rqctx: RequestContext, + query_params: Query< + PaginationParams, + >, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let pagination = pag_params.into_inner(); - let limit = rqctx.page_limit(&pagination)?; - nexus - .timeseries_schema_list(&opctx, &pagination, limit) - .await - .map(HttpResponseOk) - .map_err(HttpError::from) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let query = query_params.into_inner(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let marker = match &query.page { + WhichPage::First(..) => None, + WhichPage::Next(params::RolePage { last_seen }) => { + Some(last_seen.split_once('.').ok_or_else(|| { + Error::invalid_value( + last_seen.clone(), + "bad page token", + ) + })?) + .map(|(s1, s2)| (s1.to_string(), s2.to_string())) + } + }; + let pagparams = DataPageParams { + limit: rqctx.page_limit(&query)?, + direction: PaginationOrder::Ascending, + marker: marker.as_ref(), + }; + let roles = nexus + .roles_builtin_list(&opctx, &pagparams) + .await? + .into_iter() + .map(|i| i.into()) + .collect(); + Ok(HttpResponseOk(dropshot::ResultsPage::new( + roles, + &EmptyScanParams {}, + |role: &Role, _| params::RolePage { + last_seen: role.name.to_string(), + }, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// TODO: can we link to an OxQL reference? Do we have one? Can we even do links? - -/// Run timeseries query -/// -/// Queries are written in OxQL. -#[endpoint { - method = POST, - path = "/v1/timeseries/query", - tags = ["metrics"], -}] -async fn timeseries_query( - rqctx: RequestContext, - body: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { + async fn role_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); let nexus = &apictx.context.nexus; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let query = body.into_inner().query; - nexus - .timeseries_query(&opctx, &query) - .await - .map(|tables| HttpResponseOk(views::OxqlQueryResult { tables })) - .map_err(HttpError::from) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + let path = path_params.into_inner(); + let role_name = &path.role_name; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let role = nexus.role_builtin_fetch(&opctx, &role_name).await?; + Ok(HttpResponseOk(role.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Updates - -/// Upload TUF repository -#[endpoint { - method = PUT, - path = "/v1/system/update/repository", - tags = ["system/update"], - unpublished = true, -}] -async fn system_update_put_repository( - rqctx: RequestContext, - query: Query, - body: StreamingBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let query = query.into_inner(); - let body = body.into_stream(); - let update = - nexus.updates_put_repository(&opctx, body, query.file_name).await?; - Ok(HttpResponseOk(update)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + // Current user + + async fn current_user_view( + rqctx: RequestContext, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let nexus = &apictx.context.nexus; + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let user = nexus.silo_user_fetch_self(&opctx).await?; + let (_, silo) = nexus.current_silo_lookup(&opctx)?.fetch().await?; + Ok(HttpResponseOk(views::CurrentUser { + user: user.into(), + silo_name: silo.name().clone(), + })) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch TUF repository description -/// -/// Fetch description of TUF repository by system version. -#[endpoint { - method = GET, - path = "/v1/system/update/repository/{system_version}", - tags = ["system/update"], - unpublished = true, -}] -async fn system_update_get_repository( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let params = path_params.into_inner(); - let description = - nexus.updates_get_repository(&opctx, params.system_version).await?; - Ok(HttpResponseOk(TufRepoGetResponse { - description: description.into_external(), - })) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn current_user_groups( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let groups = nexus + .silo_user_fetch_groups_for_self( + &opctx, + &data_page_params_for(&rqctx, &query)?, + ) + .await? + .into_iter() + .map(|d| d.into()) + .collect(); + Ok(HttpResponseOk(ScanById::results_page( + &query, + groups, + &|_, group: &views::Group| group.id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Silo users - -/// List users -#[endpoint { - method = GET, - path = "/v1/users", - tags = ["silos"], -}] -async fn user_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pagparams = data_page_params_for(&rqctx, &query)?; - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let scan_params = ScanById::from_query(&query)?; + async fn current_user_ssh_key_list( + rqctx: RequestContext, + query_params: Query, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let &actor = opctx + .authn + .actor_required() + .internal_context("listing current user's ssh keys")?; + let ssh_keys = nexus + .ssh_keys_list(&opctx, actor.actor_id(), &paginated_by) + .await? + .into_iter() + .map(SshKey::from) + .collect::>(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + ssh_keys, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + async fn current_user_ssh_key_create( + rqctx: RequestContext, + new_key: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let &actor = opctx + .authn + .actor_required() + .internal_context("creating ssh key for current user")?; + let ssh_key = nexus + .ssh_key_create(&opctx, actor.actor_id(), new_key.into_inner()) + .await?; + Ok(HttpResponseCreated(ssh_key.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - // TODO: a valid UUID gets parsed here and will 404 if it doesn't exist - // (as expected) but a non-UUID string just gets let through as None - // (i.e., ignored) instead of 400ing + async fn current_user_ssh_key_view( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let &actor = opctx + .authn + .actor_required() + .internal_context("fetching one of current user's ssh keys")?; + let ssh_key_selector = params::SshKeySelector { + silo_user_id: actor.actor_id(), + ssh_key: path.ssh_key, + }; + let ssh_key_lookup = + nexus.ssh_key_lookup(&opctx, &ssh_key_selector)?; + let (.., silo_user, _, ssh_key) = ssh_key_lookup.fetch().await?; + // Ensure the SSH key exists in the current silo + assert_eq!(silo_user.id(), actor.actor_id()); + Ok(HttpResponseOk(ssh_key.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } - let users = if let Some(group_id) = scan_params.selector.group { + async fn current_user_ssh_key_delete( + rqctx: RequestContext, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let &actor = opctx + .authn + .actor_required() + .internal_context("deleting one of current user's ssh keys")?; + let ssh_key_selector = params::SshKeySelector { + silo_user_id: actor.actor_id(), + ssh_key: path.ssh_key, + }; + let ssh_key_lookup = + nexus.ssh_key_lookup(&opctx, &ssh_key_selector)?; nexus - .current_silo_group_users_list(&opctx, &pagparams, &group_id) - .await? - } else { - nexus.silo_users_list_current(&opctx, &pagparams).await? - }; - - Ok(HttpResponseOk(ScanById::results_page( - &query, - users.into_iter().map(|i| i.into()).collect(), - &|_, user: &User| user.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + .ssh_key_delete(&opctx, actor.actor_id(), &ssh_key_lookup) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Silo groups - -/// List groups -#[endpoint { - method = GET, - path = "/v1/groups", - tags = ["silos"], -}] -async fn group_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pagparams = data_page_params_for(&rqctx, &query)?; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let groups = nexus - .silo_groups_list(&opctx, &pagparams) - .await? - .into_iter() - .map(|i| i.into()) - .collect(); - Ok(HttpResponseOk(ScanById::results_page( - &query, - groups, - &|_, group: &Group| group.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn probe_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + + let probes = nexus + .probe_list(&opctx, &project_lookup, &paginated_by) + .await?; -/// Fetch group -#[endpoint { - method = GET, - path = "/v1/groups/{group_id}", - tags = ["silos"], -}] -async fn group_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let (.., group) = - nexus.silo_group_lookup(&opctx, &path.group_id).fetch().await?; - Ok(HttpResponseOk(group.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + probes, + &|_, p: &ProbeInfo| match paginated_by { + PaginatedBy::Id(_) => NameOrId::Id(p.id), + PaginatedBy::Name(_) => NameOrId::Name(p.name.clone()), + }, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Built-in (system) users - -/// List built-in users -#[endpoint { - method = GET, - path = "/v1/system/users-builtin", - tags = ["system/silos"], -}] -async fn user_builtin_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pagparams = - data_page_params_for(&rqctx, &query)?.map_name(|n| Name::ref_cast(n)); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let users = nexus - .users_builtin_list(&opctx, &pagparams) - .await? - .into_iter() - .map(|i| i.into()) - .collect(); - Ok(HttpResponseOk(ScanByName::results_page( - &query, - users, - &marker_for_name, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn probe_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let project_selector = query_params.into_inner(); + let project_lookup = + nexus.project_lookup(&opctx, project_selector)?; + let probe = + nexus.probe_get(&opctx, &project_lookup, &path.probe).await?; + Ok(HttpResponseOk(probe)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -/// Fetch built-in user -#[endpoint { - method = GET, - path = "/v1/system/users-builtin/{user}", - tags = ["system/silos"], -}] -async fn user_builtin_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let nexus = &apictx.context.nexus; - let user_selector = path_params.into_inner(); - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let (.., user) = - nexus.user_builtin_lookup(&opctx, &user_selector)?.fetch().await?; - Ok(HttpResponseOk(user.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn probe_create( + rqctx: RequestContext, + query_params: Query, + new_probe: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + + let nexus = &apictx.context.nexus; + let new_probe_params = &new_probe.into_inner(); + let project_selector = query_params.into_inner(); + let project_lookup = + nexus.project_lookup(&opctx, project_selector)?; + let probe = nexus + .probe_create(&opctx, &project_lookup, &new_probe_params) + .await?; + Ok(HttpResponseCreated(probe.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Built-in roles + async fn probe_delete( + rqctx: RequestContext, + query_params: Query, + path_params: Path, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let project_selector = query_params.into_inner(); + let project_lookup = + nexus.project_lookup(&opctx, project_selector)?; + nexus.probe_delete(&opctx, &project_lookup, path.probe).await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } -// Roles have their own pagination scheme because they do not use the usual "id" -// or "name" types. For more, see the comment in dbinit.sql. -#[derive(Deserialize, JsonSchema, Serialize)] -struct RolePage { - last_seen: String, -} + async fn login_saml_begin( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + console_api::login_saml_begin(rqctx, path_params, query_params).await + } -/// Path parameters for global (system) role requests -#[derive(Deserialize, JsonSchema)] -struct RolePathParam { - /// The built-in role's unique name. - role_name: String, -} + async fn login_saml_redirect( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + console_api::login_saml_redirect(rqctx, path_params, query_params).await + } -/// List built-in roles -#[endpoint { - method = GET, - path = "/v1/system/roles", - tags = ["roles"], -}] -async fn role_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let marker = match &query.page { - WhichPage::First(..) => None, - WhichPage::Next(RolePage { last_seen }) => { - Some(last_seen.split_once('.').ok_or_else(|| { - Error::invalid_value(last_seen.clone(), "bad page token") - })?) - .map(|(s1, s2)| (s1.to_string(), s2.to_string())) - } - }; - let pagparams = DataPageParams { - limit: rqctx.page_limit(&query)?, - direction: PaginationOrder::Ascending, - marker: marker.as_ref(), - }; - let roles = nexus - .roles_builtin_list(&opctx, &pagparams) - .await? - .into_iter() - .map(|i| i.into()) - .collect(); - Ok(HttpResponseOk(dropshot::ResultsPage::new( - roles, - &EmptyScanParams {}, - |role: &Role, _| RolePage { last_seen: role.name.to_string() }, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn login_saml( + rqctx: RequestContext, + path_params: Path, + body_bytes: dropshot::UntypedBody, + ) -> Result { + console_api::login_saml(rqctx, path_params, body_bytes).await + } -/// Fetch built-in role -#[endpoint { - method = GET, - path = "/v1/system/roles/{role_name}", - tags = ["roles"], -}] -async fn role_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let role_name = &path.role_name; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let role = nexus.role_builtin_fetch(&opctx, &role_name).await?; - Ok(HttpResponseOk(role.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn login_local_begin( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + console_api::login_local_begin(rqctx, path_params, query_params).await + } -// Current user - -/// Fetch user for current session -#[endpoint { - method = GET, - path = "/v1/me", - tags = ["session"], -}] -pub(crate) async fn current_user_view( - rqctx: RequestContext, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let nexus = &apictx.context.nexus; - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let user = nexus.silo_user_fetch_self(&opctx).await?; - let (_, silo) = nexus.current_silo_lookup(&opctx)?.fetch().await?; - Ok(HttpResponseOk(views::CurrentUser { - user: user.into(), - silo_name: silo.name().clone(), - })) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn login_local( + rqctx: RequestContext, + path_params: Path, + credentials: TypedBody, + ) -> Result, HttpError> + { + console_api::login_local(rqctx, path_params, credentials).await + } -/// Fetch current user's groups -#[endpoint { - method = GET, - path = "/v1/me/groups", - tags = ["session"], - }] -pub(crate) async fn current_user_groups( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let groups = nexus - .silo_user_fetch_groups_for_self( - &opctx, - &data_page_params_for(&rqctx, &query)?, - ) - .await? - .into_iter() - .map(|d| d.into()) - .collect(); - Ok(HttpResponseOk(ScanById::results_page( - &query, - groups, - &|_, group: &views::Group| group.id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn logout( + rqctx: RequestContext, + cookies: Cookies, + ) -> Result, HttpError> + { + console_api::logout(rqctx, cookies).await + } -// Per-user SSH public keys - -/// List SSH public keys -/// -/// Lists SSH public keys for the currently authenticated user. -#[endpoint { - method = GET, - path = "/v1/me/ssh-keys", - tags = ["session"], -}] -async fn current_user_ssh_key_list( - rqctx: RequestContext, - query_params: Query, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let &actor = opctx - .authn - .actor_required() - .internal_context("listing current user's ssh keys")?; - let ssh_keys = nexus - .ssh_keys_list(&opctx, actor.actor_id(), &paginated_by) - .await? - .into_iter() - .map(SshKey::from) - .collect::>(); - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - ssh_keys, - &marker_for_name_or_id, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn login_begin( + rqctx: RequestContext, + query_params: Query, + ) -> Result { + console_api::login_begin(rqctx, query_params).await + } -/// Create SSH public key -/// -/// Create an SSH public key for the currently authenticated user. -#[endpoint { - method = POST, - path = "/v1/me/ssh-keys", - tags = ["session"], -}] -async fn current_user_ssh_key_create( - rqctx: RequestContext, - new_key: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let &actor = opctx - .authn - .actor_required() - .internal_context("creating ssh key for current user")?; - let ssh_key = nexus - .ssh_key_create(&opctx, actor.actor_id(), new_key.into_inner()) - .await?; - Ok(HttpResponseCreated(ssh_key.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn console_projects( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + console_api::console_projects(rqctx, path_params).await + } -/// Fetch SSH public key -/// -/// Fetch SSH public key associated with the currently authenticated user. -#[endpoint { - method = GET, - path = "/v1/me/ssh-keys/{ssh_key}", - tags = ["session"], -}] -async fn current_user_ssh_key_view( - rqctx: RequestContext, - path_params: Path, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let &actor = opctx - .authn - .actor_required() - .internal_context("fetching one of current user's ssh keys")?; - let ssh_key_selector = params::SshKeySelector { - silo_user_id: actor.actor_id(), - ssh_key: path.ssh_key, - }; - let ssh_key_lookup = nexus.ssh_key_lookup(&opctx, &ssh_key_selector)?; - let (.., silo_user, _, ssh_key) = ssh_key_lookup.fetch().await?; - // Ensure the SSH key exists in the current silo - assert_eq!(silo_user.id(), actor.actor_id()); - Ok(HttpResponseOk(ssh_key.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn console_settings_page( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + console_api::console_settings_page(rqctx, path_params).await + } -/// Delete SSH public key -/// -/// Delete an SSH public key associated with the currently authenticated user. -#[endpoint { - method = DELETE, - path = "/v1/me/ssh-keys/{ssh_key}", - tags = ["session"], -}] -async fn current_user_ssh_key_delete( - rqctx: RequestContext, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let &actor = opctx - .authn - .actor_required() - .internal_context("deleting one of current user's ssh keys")?; - let ssh_key_selector = params::SshKeySelector { - silo_user_id: actor.actor_id(), - ssh_key: path.ssh_key, - }; - let ssh_key_lookup = nexus.ssh_key_lookup(&opctx, &ssh_key_selector)?; - nexus.ssh_key_delete(&opctx, actor.actor_id(), &ssh_key_lookup).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn console_system_page( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + console_api::console_system_page(rqctx, path_params).await + } -/// List instrumentation probes -#[endpoint { - method = GET, - path = "/v1/probes", - tags = ["system/probes"], -}] -async fn probe_list( - rqctx: RequestContext, - query_params: Query>, -) -> Result>, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + async fn console_lookup( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + console_api::console_lookup(rqctx, path_params).await + } - let nexus = &apictx.context.nexus; - let query = query_params.into_inner(); - let pag_params = data_page_params_for(&rqctx, &query)?; - let scan_params = ScanByNameOrId::from_query(&query)?; - let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; - let project_lookup = - nexus.project_lookup(&opctx, scan_params.selector.clone())?; - - let probes = - nexus.probe_list(&opctx, &project_lookup, &paginated_by).await?; - - Ok(HttpResponseOk(ScanByNameOrId::results_page( - &query, - probes, - &|_, p: &ProbeInfo| match paginated_by { - PaginatedBy::Id(_) => NameOrId::Id(p.id), - PaginatedBy::Name(_) => NameOrId::Name(p.name.clone()), - }, - )?)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn console_root( + rqctx: RequestContext, + ) -> Result, HttpError> { + console_api::console_root(rqctx).await + } -/// View instrumentation probe -#[endpoint { - method = GET, - path = "/v1/probes/{probe}", - tags = ["system/probes"], -}] -async fn probe_view( - rqctx: RequestContext, - path_params: Path, - query_params: Query, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?; + async fn console_projects_new( + rqctx: RequestContext, + ) -> Result, HttpError> { + console_api::console_projects_new(rqctx).await + } - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let project_selector = query_params.into_inner(); - let project_lookup = nexus.project_lookup(&opctx, project_selector)?; - let probe = - nexus.probe_get(&opctx, &project_lookup, &path.probe).await?; - Ok(HttpResponseOk(probe)) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn console_silo_images( + rqctx: RequestContext, + ) -> Result, HttpError> { + console_api::console_silo_images(rqctx).await + } -/// Create instrumentation probe -#[endpoint { - method = POST, - path = "/v1/probes", - tags = ["system/probes"], -}] -async fn probe_create( - rqctx: RequestContext, - query_params: Query, - new_probe: TypedBody, -) -> Result, HttpError> { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + async fn console_silo_utilization( + rqctx: RequestContext, + ) -> Result, HttpError> { + console_api::console_silo_utilization(rqctx).await + } - let nexus = &apictx.context.nexus; - let new_probe_params = &new_probe.into_inner(); - let project_selector = query_params.into_inner(); - let project_lookup = nexus.project_lookup(&opctx, project_selector)?; - let probe = nexus - .probe_create(&opctx, &project_lookup, &new_probe_params) - .await?; - Ok(HttpResponseCreated(probe.into())) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn console_silo_access( + rqctx: RequestContext, + ) -> Result, HttpError> { + console_api::console_silo_access(rqctx).await + } -/// Delete instrumentation probe -#[endpoint { - method = DELETE, - path = "/v1/probes/{probe}", - tags = ["system/probes"], -}] -async fn probe_delete( - rqctx: RequestContext, - query_params: Query, - path_params: Path, -) -> Result { - let apictx = rqctx.context(); - let handler = async { - let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + async fn asset( + rqctx: RequestContext, + path_params: Path, + ) -> Result, HttpError> { + console_api::asset(rqctx, path_params).await + } - let nexus = &apictx.context.nexus; - let path = path_params.into_inner(); - let project_selector = query_params.into_inner(); - let project_lookup = nexus.project_lookup(&opctx, project_selector)?; - nexus.probe_delete(&opctx, &project_lookup, path.probe).await?; - Ok(HttpResponseDeleted()) - }; - apictx - .context - .external_latencies - .instrument_dropshot_handler(&rqctx, handler) - .await -} + async fn device_auth_request( + rqctx: RequestContext, + params: TypedBody, + ) -> Result, HttpError> { + device_auth::device_auth_request(rqctx, params).await + } + + async fn device_auth_verify( + rqctx: RequestContext, + ) -> Result, HttpError> { + device_auth::device_auth_verify(rqctx).await + } + + async fn device_auth_success( + rqctx: RequestContext, + ) -> Result, HttpError> { + device_auth::device_auth_success(rqctx).await + } -#[cfg(test)] -mod test { - use super::external_api; + async fn device_auth_confirm( + rqctx: RequestContext, + params: TypedBody, + ) -> Result { + device_auth::device_auth_confirm(rqctx, params).await + } - #[test] - fn test_nexus_tag_policy() { - // This will fail if any of the endpoints don't match the policy in - // ./tag-config.json - let _ = external_api(); + async fn device_access_token( + rqctx: RequestContext, + params: TypedBody, + ) -> Result, HttpError> { + device_auth::device_access_token(rqctx, params.into_inner()).await } } diff --git a/nexus/src/external_api/tag-config.json b/nexus/src/external_api/tag-config.json deleted file mode 100644 index 6974906507..0000000000 --- a/nexus/src/external_api/tag-config.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "allow_other_tags": false, - "endpoint_tag_policy": "ExactlyOne", - "tag_definitions": { - "disks": { - "description": "Virtual disks are used to store instance-local data which includes the operating system.", - "external_docs": { - "url": "http://docs.oxide.computer/api/disks" - } - }, - "floating-ips": { - "description": "Floating IPs allow a project to allocate well-known IPs to instances.", - "external_docs": { - "url": "http://docs.oxide.computer/api/floating-ips" - } - }, - "hidden": { - "description": "TODO operations that will not ship to customers", - "external_docs": { - "url": "http://docs.oxide.computer/api" - } - }, - "images": { - "description": "Images are read-only virtual disks that may be used to boot virtual machines.", - "external_docs": { - "url": "http://docs.oxide.computer/api/images" - } - }, - "instances": { - "description": "Virtual machine instances are the basic unit of computation. These operations are used for provisioning, controlling, and destroying instances.", - "external_docs": { - "url": "http://docs.oxide.computer/api/instances" - } - }, - "login": { - "description": "Authentication endpoints", - "external_docs": { - "url": "http://docs.oxide.computer/api/login" - } - }, - "metrics": { - "description": "Silo-scoped metrics", - "external_docs": { - "url": "http://docs.oxide.computer/api/metrics" - } - }, - "policy": { - "description": "System-wide IAM policy", - "external_docs": { - "url": "http://docs.oxide.computer/api/policy" - } - }, - "projects": { - "description": "Projects are a grouping of associated resources such as instances and disks within a silo for purposes of billing and access control.", - "external_docs": { - "url": "http://docs.oxide.computer/api/projects" - } - }, - "roles": { - "description": "Roles are a component of Identity and Access Management (IAM) that allow a user or agent account access to additional permissions.", - "external_docs": { - "url": "http://docs.oxide.computer/api/roles" - } - }, - "session": { - "description": "Information pertaining to the current session.", - "external_docs": { - "url": "http://docs.oxide.computer/api/session" - } - }, - "silos": { - "description": "Silos represent a logical partition of users and resources.", - "external_docs": { - "url": "http://docs.oxide.computer/api/silos" - } - }, - "snapshots": { - "description": "Snapshots of virtual disks at a particular point in time.", - "external_docs": { - "url": "http://docs.oxide.computer/api/snapshots" - } - }, - "vpcs": { - "description": "Virtual Private Clouds (VPCs) provide isolated network environments for managing and deploying services.", - "external_docs": { - "url": "http://docs.oxide.computer/api/vpcs" - } - }, - "system/probes": { - "description": "Probes for testing network connectivity", - "external_docs": { - "url": "http://docs.oxide.computer/api/probes" - } - }, - "system/status": { - "description": "Endpoints related to system health", - "external_docs": { - "url": "http://docs.oxide.computer/api/system-status" - } - }, - "system/hardware": { - "description": "These operations pertain to hardware inventory and management. Racks are the unit of expansion of an Oxide deployment. Racks are in turn composed of sleds, switches, power supplies, and a cabled backplane.", - "external_docs": { - "url": "http://docs.oxide.computer/api/system-hardware" - } - }, - "system/metrics": { - "description": "Metrics provide insight into the operation of the Oxide deployment. These include telemetry on hardware and software components that can be used to understand the current state as well as to diagnose issues.", - "external_docs": { - "url": "http://docs.oxide.computer/api/system-metrics" - } - }, - "system/networking": { - "description": "This provides rack-level network configuration.", - "external_docs": { - "url": "http://docs.oxide.computer/api/system-networking" - } - }, - "system/silos": { - "description": "Silos represent a logical partition of users and resources.", - "external_docs": { - "url": "http://docs.oxide.computer/api/system-silos" - } - } - } -} diff --git a/nexus/src/internal_api/http_entrypoints.rs b/nexus/src/internal_api/http_entrypoints.rs index 9965b6e21e..66a8090f11 100644 --- a/nexus/src/internal_api/http_entrypoints.rs +++ b/nexus/src/internal_api/http_entrypoints.rs @@ -52,7 +52,7 @@ use omicron_common::api::internal::nexus::ProducerRegistrationResponse; use omicron_common::api::internal::nexus::RepairFinishInfo; use omicron_common::api::internal::nexus::RepairProgress; use omicron_common::api::internal::nexus::RepairStartInfo; -use omicron_common::api::internal::nexus::SledInstanceState; +use omicron_common::api::internal::nexus::SledVmmState; use omicron_common::update::ArtifactId; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::InstanceUuid; @@ -168,8 +168,8 @@ impl NexusInternalApi for NexusInternalApiImpl { async fn cpapi_instances_put( rqctx: RequestContext, - path_params: Path, - new_runtime_state: TypedBody, + path_params: Path, + new_runtime_state: TypedBody, ) -> Result { let apictx = &rqctx.context().context; let nexus = &apictx.nexus; @@ -178,11 +178,7 @@ impl NexusInternalApi for NexusInternalApiImpl { let opctx = crate::context::op_context_for_internal_api(&rqctx).await; let handler = async { nexus - .notify_instance_updated( - &opctx, - InstanceUuid::from_untyped_uuid(path.instance_id), - &new_state, - ) + .notify_vmm_updated(&opctx, path.propolis_id, &new_state) .await?; Ok(HttpResponseUpdatedNoContent()) }; diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index d5c853b15b..eaabbd748b 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -53,18 +53,6 @@ use uuid::Uuid; #[macro_use] extern crate slog; -/// Run the OpenAPI generator for the external API, which emits the OpenAPI spec -/// to stdout. -pub fn run_openapi_external() -> Result<(), String> { - external_api() - .openapi("Oxide Region API", "20240821.0") - .description("API for interacting with the Oxide control plane") - .contact_url("https://oxide.computer") - .contact_email("api@oxide.computer") - .write(&mut std::io::stdout()) - .map_err(|e| e.to_string()) -} - /// A partially-initialized Nexus server, which exposes an internal interface, /// but is not ready to receive external requests. pub struct InternalServer { @@ -384,12 +372,7 @@ impl nexus_test_interface::NexusServer for Server { self.apictx .context .nexus - .upsert_dataset( - dataset_id, - zpool_id, - address, - nexus_db_queries::db::model::DatasetKind::Crucible, - ) + .upsert_crucible_dataset(dataset_id, zpool_id, address) .await .unwrap(); } diff --git a/nexus/src/populate.rs b/nexus/src/populate.rs index 4fcb126356..f026b1b504 100644 --- a/nexus/src/populate.rs +++ b/nexus/src/populate.rs @@ -380,7 +380,7 @@ mod test { let logctx = dev::test_setup_log("test_populator"); let mut db = test_setup_database(&logctx.log).await; let cfg = db::Config { url: db.pg_config().clone() }; - let pool = Arc::new(db::Pool::new(&logctx.log, &cfg)); + let pool = Arc::new(db::Pool::new_single_host(&logctx.log, &cfg)); let datastore = Arc::new( db::DataStore::new(&logctx.log, pool, None).await.unwrap(), ); @@ -422,19 +422,13 @@ mod test { }) .unwrap(); - // Test again with the database offline. In principle we could do this - // immediately without creating a new pool and datastore. However, the - // pool's default behavior is to wait 30 seconds for a connection, which - // makes this test take a long time. (See the note in - // nexus/src/db/pool.rs about this.) So let's create a pool with an - // arbitrarily short timeout now. (We wouldn't want to do this above - // because we do want to wait a bit when we expect things to work, in - // case the test system is busy.) + // Test again with the database offline. In principle we could do this + // immediately without creating a new pool and datastore. // - // Anyway, if we try again with a broken database, we should get a + // If we try again with a broken database, we should get a // ServiceUnavailable error, which indicates a transient failure. let pool = - Arc::new(db::Pool::new_failfast_for_tests(&logctx.log, &cfg)); + Arc::new(db::Pool::new_single_host_failfast(&logctx.log, &cfg)); // We need to create the datastore before tearing down the database, as // it verifies the schema version of the DB while booting. let datastore = Arc::new( diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index 50110ecaca..ff89af7c6c 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -25,6 +25,7 @@ http.workspace = true hyper.workspace = true illumos-utils.workspace = true internal-dns.workspace = true +nexus-client.workspace = true nexus-config.workspace = true nexus-db-queries.workspace = true nexus-sled-agent-shared.workspace = true diff --git a/nexus/test-utils/src/background.rs b/nexus/test-utils/src/background.rs new file mode 100644 index 0000000000..58792e547d --- /dev/null +++ b/nexus/test-utils/src/background.rs @@ -0,0 +1,86 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Helper functions related to Nexus background tasks + +use crate::http_testing::NexusRequest; +use dropshot::test_util::ClientTestContext; +use nexus_client::types::BackgroundTask; +use nexus_client::types::CurrentStatus; +use nexus_client::types::CurrentStatusRunning; +use nexus_client::types::LastResult; +use nexus_client::types::LastResultCompleted; +use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; +use std::time::Duration; + +/// Return the most recent start time for a background task +fn most_recent_start_time( + task: &BackgroundTask, +) -> Option> { + match task.current { + CurrentStatus::Idle => match task.last { + LastResult::Completed(LastResultCompleted { + start_time, .. + }) => Some(start_time), + LastResult::NeverCompleted => None, + }, + CurrentStatus::Running(CurrentStatusRunning { start_time, .. }) => { + Some(start_time) + } + } +} + +/// Given the name of a background task, activate it, then wait for it to +/// complete. Return the last polled `BackgroundTask` object. +pub async fn activate_background_task( + internal_client: &ClientTestContext, + task_name: &str, +) -> BackgroundTask { + let task = NexusRequest::object_get( + internal_client, + &format!("/bgtasks/view/{task_name}"), + ) + .execute_and_parse_unwrap::() + .await; + + let last_start = most_recent_start_time(&task); + + internal_client + .make_request( + http::Method::POST, + "/bgtasks/activate", + Some(serde_json::json!({ + "bgtask_names": vec![String::from(task_name)] + })), + http::StatusCode::NO_CONTENT, + ) + .await + .unwrap(); + + // Wait for the task to finish + let last_task_poll = wait_for_condition( + || async { + let task = NexusRequest::object_get( + internal_client, + &format!("/bgtasks/view/{task_name}"), + ) + .execute_and_parse_unwrap::() + .await; + + if matches!(&task.current, CurrentStatus::Idle) + && most_recent_start_time(&task) > last_start + { + Ok(task) + } else { + Err(CondCheckError::<()>::NotYet) + } + }, + &Duration::from_millis(500), + &Duration::from_secs(60), + ) + .await + .unwrap(); + + last_task_poll +} diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index acee46ce10..c3efbb83cf 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -84,6 +84,7 @@ use uuid::Uuid; pub use sim::TEST_HARDWARE_THREADS; pub use sim::TEST_RESERVOIR_RAM; +pub mod background; pub mod db; pub mod http_testing; pub mod resource_helpers; diff --git a/nexus/tests/integration_tests/commands.rs b/nexus/tests/integration_tests/commands.rs index c2277ba776..7ae4223556 100644 --- a/nexus/tests/integration_tests/commands.rs +++ b/nexus/tests/integration_tests/commands.rs @@ -15,10 +15,7 @@ use omicron_test_utils::dev::test_cmds::path_to_executable; use omicron_test_utils::dev::test_cmds::run_command; use omicron_test_utils::dev::test_cmds::temp_file_path; use omicron_test_utils::dev::test_cmds::EXIT_FAILURE; -use omicron_test_utils::dev::test_cmds::EXIT_SUCCESS; use omicron_test_utils::dev::test_cmds::EXIT_USAGE; -use openapiv3::OpenAPI; -use std::collections::BTreeMap; use std::fs; use std::path::PathBuf; use subprocess::Exec; @@ -78,105 +75,3 @@ fn test_nexus_invalid_config() { ); assert!(&stderr_text.starts_with(&expected_err)); } - -#[track_caller] -fn run_command_with_arg(arg: &str) -> (String, String) { - // This is a little goofy: we need a config file for the program. - // (Arguably, --openapi shouldn't require a config file, but it's - // conceivable that the API metadata or the exposed endpoints would depend - // on the configuration.) We ship a config file in "examples", and we may - // as well use it here -- it would be a bug if that one didn't work for this - // purpose. However, it's not clear how to reliably locate it at runtime. - // But we do know where it is at compile time, so we load it then. - let config = include_str!("../../examples/config.toml"); - let config_path = write_config(config); - let exec = Exec::cmd(path_to_nexus()).arg(&config_path).arg(arg); - let (exit_status, stdout_text, stderr_text) = run_command(exec); - fs::remove_file(&config_path).expect("failed to remove temporary file"); - assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text); - - (stdout_text, stderr_text) -} - -#[test] -fn test_nexus_openapi() { - let (stdout_text, stderr_text) = run_command_with_arg("--openapi"); - assert_contents("tests/output/cmd-nexus-openapi-stderr", &stderr_text); - - // Make sure the result parses as a valid OpenAPI spec and sanity-check a - // few fields. - let spec: OpenAPI = serde_json::from_str(&stdout_text) - .expect("stdout was not valid OpenAPI"); - assert_eq!(spec.openapi, "3.0.3"); - assert_eq!(spec.info.title, "Oxide Region API"); - assert_eq!(spec.info.version, "20240821.0"); - - // Spot check a couple of items. - assert!(!spec.paths.paths.is_empty()); - assert!(spec.paths.paths.get("/v1/projects").is_some()); - - // Check for lint errors. - let errors = openapi_lint::validate_external(&spec); - assert!(errors.is_empty(), "{}", errors.join("\n\n")); - - // Construct a string that helps us identify the organization of tags and - // operations. - let mut ops_by_tag = - BTreeMap::>::new(); - for (path, method, op) in spec.operations() { - // Make sure each operation has exactly one tag. Note, we intentionally - // do this before validating the OpenAPI output as fixing an error here - // would necessitate refreshing the spec file again. - assert_eq!( - op.tags.len(), - 1, - "operation '{}' has {} tags rather than 1", - op.operation_id.as_ref().unwrap(), - op.tags.len() - ); - - // Every non-hidden endpoint must have a summary - if !op.tags.contains(&"hidden".to_string()) { - assert!( - op.summary.is_some(), - "operation '{}' is missing a summary doc comment", - op.operation_id.as_ref().unwrap() - ); - } - - ops_by_tag - .entry(op.tags.first().unwrap().to_string()) - .or_default() - .push(( - op.operation_id.as_ref().unwrap().to_string(), - method.to_string().to_uppercase(), - path.to_string(), - )); - } - - let mut tags = String::new(); - for (tag, mut ops) in ops_by_tag { - ops.sort(); - tags.push_str(&format!(r#"API operations found with tag "{}""#, tag)); - tags.push_str(&format!( - "\n{:40} {:8} {}\n", - "OPERATION ID", "METHOD", "URL PATH" - )); - for (operation_id, method, path) in ops { - tags.push_str(&format!( - "{:40} {:8} {}\n", - operation_id, method, path - )); - } - tags.push('\n'); - } - - // Confirm that the output hasn't changed. It's expected that we'll change - // this file as the API evolves, but pay attention to the diffs to ensure - // that the changes match your expectations. - assert_contents("../openapi/nexus.json", &stdout_text); - - // When this fails, verify that operations on which you're adding, - // renaming, or changing the tags are what you intend. - assert_contents("tests/output/nexus_tags.txt", &tags); -} diff --git a/nexus/tests/integration_tests/device_auth.rs b/nexus/tests/integration_tests/device_auth.rs index 5bb34eb19e..65730f6cc8 100644 --- a/nexus/tests/integration_tests/device_auth.rs +++ b/nexus/tests/integration_tests/device_auth.rs @@ -4,11 +4,11 @@ use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder}; use nexus_test_utils_macros::nexus_test; -use nexus_types::external_api::views::{ - DeviceAccessTokenGrant, DeviceAccessTokenType, DeviceAuthResponse, -}; -use omicron_nexus::external_api::device_auth::{ - DeviceAccessTokenRequest, DeviceAuthRequest, DeviceAuthVerify, +use nexus_types::external_api::{ + params::{DeviceAccessTokenRequest, DeviceAuthRequest, DeviceAuthVerify}, + views::{ + DeviceAccessTokenGrant, DeviceAccessTokenType, DeviceAuthResponse, + }, }; use http::{header, method::Method, StatusCode}; diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index 234ab5f382..fe6aab2770 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -188,12 +188,13 @@ async fn set_instance_state( } async fn instance_simulate(nexus: &Arc, id: &InstanceUuid) { - let sa = nexus - .instance_sled_by_id(id) + let info = nexus + .active_instance_info(id, None) .await .unwrap() .expect("instance must be on a sled to simulate a state change"); - sa.instance_finish_transition(id.into_untyped_uuid()).await; + + info.sled_client.vmm_finish_transition(info.propolis_id).await; } #[nexus_test] diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index eb3c88eb38..a7228e0841 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -780,12 +780,13 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { let instance_next = instance_get(&client, &instance_url).await; assert_eq!(instance_next.runtime.run_state, InstanceState::Running); - let original_sled = nexus - .instance_sled_id(&instance_id) + let sled_info = nexus + .active_instance_info(&instance_id, None) .await .unwrap() .expect("running instance should have a sled"); + let original_sled = sled_info.sled_id; let dst_sled_id = if original_sled == default_sled_id { other_sled_id } else { @@ -808,12 +809,13 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { .parsed_body::() .unwrap(); - let current_sled = nexus - .instance_sled_id(&instance_id) + let new_sled_info = nexus + .active_instance_info(&instance_id, None) .await .unwrap() .expect("running instance should have a sled"); + let current_sled = new_sled_info.sled_id; assert_eq!(current_sled, original_sled); // Ensure that both sled agents report that the migration is in progress. @@ -840,6 +842,15 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { assert_eq!(migration.target_state, MigrationState::Pending.into()); assert_eq!(migration.source_state, MigrationState::Pending.into()); + let info = nexus + .active_instance_info(&instance_id, None) + .await + .unwrap() + .expect("instance should be on a sled"); + let src_propolis_id = info.propolis_id; + let dst_propolis_id = + info.dst_propolis_id.expect("instance should have a migration target"); + // Simulate the migration. We will use `instance_single_step_on_sled` to // single-step both sled-agents through the migration state machine and // ensure that the migration state looks nice at each step. @@ -847,15 +858,15 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { cptestctx, nexus, original_sled, - instance_id, + src_propolis_id, migration_id, ) .await; // Move source to "migrating". - instance_single_step_on_sled(cptestctx, nexus, original_sled, instance_id) + vmm_single_step_on_sled(cptestctx, nexus, original_sled, src_propolis_id) .await; - instance_single_step_on_sled(cptestctx, nexus, original_sled, instance_id) + vmm_single_step_on_sled(cptestctx, nexus, original_sled, src_propolis_id) .await; let migration = dbg!(migration_fetch(cptestctx, migration_id).await); @@ -865,9 +876,9 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { assert_eq!(instance.runtime.run_state, InstanceState::Migrating); // Move target to "migrating". - instance_single_step_on_sled(cptestctx, nexus, dst_sled_id, instance_id) + vmm_single_step_on_sled(cptestctx, nexus, dst_sled_id, dst_propolis_id) .await; - instance_single_step_on_sled(cptestctx, nexus, dst_sled_id, instance_id) + vmm_single_step_on_sled(cptestctx, nexus, dst_sled_id, dst_propolis_id) .await; let migration = dbg!(migration_fetch(cptestctx, migration_id).await); @@ -877,7 +888,7 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { assert_eq!(instance.runtime.run_state, InstanceState::Migrating); // Move the source to "completed" - instance_simulate_on_sled(cptestctx, nexus, original_sled, instance_id) + vmm_simulate_on_sled(cptestctx, nexus, original_sled, src_propolis_id) .await; let migration = dbg!(migration_fetch(cptestctx, migration_id).await); @@ -887,15 +898,16 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { assert_eq!(instance.runtime.run_state, InstanceState::Migrating); // Move the target to "completed". - instance_simulate_on_sled(cptestctx, nexus, dst_sled_id, instance_id).await; + vmm_simulate_on_sled(cptestctx, nexus, dst_sled_id, dst_propolis_id).await; instance_wait_for_state(&client, instance_id, InstanceState::Running).await; let current_sled = nexus - .instance_sled_id(&instance_id) + .active_instance_info(&instance_id, None) .await .unwrap() - .expect("migrated instance should still have a sled"); + .expect("migrated instance should still have a sled") + .sled_id; assert_eq!(current_sled, dst_sled_id); @@ -978,11 +990,13 @@ async fn test_instance_migrate_v2p_and_routes( .derive_guest_network_interface_info(&opctx, &authz_instance) .await .unwrap(); + let original_sled_id = nexus - .instance_sled_id(&instance_id) + .active_instance_info(&instance_id, None) .await .unwrap() - .expect("running instance should have a sled"); + .expect("running instance should have a sled") + .sled_id; let mut sled_agents = vec![cptestctx.sled_agent.sled_agent.clone()]; sled_agents.extend(other_sleds.iter().map(|tup| tup.1.sled_agent.clone())); @@ -1035,25 +1049,35 @@ async fn test_instance_migrate_v2p_and_routes( .expect("since we've started a migration, the instance record must have a migration id!") }; + let info = nexus + .active_instance_info(&instance_id, None) + .await + .unwrap() + .expect("instance should be on a sled"); + let src_propolis_id = info.propolis_id; + let dst_propolis_id = + info.dst_propolis_id.expect("instance should have a migration target"); + // Tell both sled-agents to pretend to do the migration. instance_simulate_migration_source( cptestctx, nexus, original_sled_id, - instance_id, + src_propolis_id, migration_id, ) .await; - instance_simulate_on_sled(cptestctx, nexus, original_sled_id, instance_id) + vmm_simulate_on_sled(cptestctx, nexus, original_sled_id, src_propolis_id) .await; - instance_simulate_on_sled(cptestctx, nexus, dst_sled_id, instance_id).await; + vmm_simulate_on_sled(cptestctx, nexus, dst_sled_id, dst_propolis_id).await; instance_wait_for_state(&client, instance_id, InstanceState::Running).await; let current_sled = nexus - .instance_sled_id(&instance_id) + .active_instance_info(&instance_id, None) .await .unwrap() - .expect("migrated instance should have a sled"); + .expect("migrated instance should have a sled") + .sled_id; assert_eq!(current_sled, dst_sled_id); for sled_agent in &sled_agents { @@ -1373,10 +1397,11 @@ async fn test_instance_metrics_with_migration( // Request migration to the other sled. This reserves resources on the // target sled, but shouldn't change the virtual provisioning counters. let original_sled = nexus - .instance_sled_id(&instance_id) + .active_instance_info(&instance_id, None) .await .unwrap() - .expect("running instance should have a sled"); + .expect("running instance should have a sled") + .sled_id; let dst_sled_id = if original_sled == default_sled_id { other_sled_id @@ -1420,6 +1445,15 @@ async fn test_instance_metrics_with_migration( .expect("since we've started a migration, the instance record must have a migration id!") }; + let info = nexus + .active_instance_info(&instance_id, None) + .await + .unwrap() + .expect("instance should be on a sled"); + let src_propolis_id = info.propolis_id; + let dst_propolis_id = + info.dst_propolis_id.expect("instance should have a migration target"); + // Wait for the instance to be in the `Migrating` state. Otherwise, the // subsequent `instance_wait_for_state(..., Running)` may see the `Running` // state from the *old* VMM, rather than waiting for the migration to @@ -1428,13 +1462,13 @@ async fn test_instance_metrics_with_migration( cptestctx, nexus, original_sled, - instance_id, + src_propolis_id, migration_id, ) .await; - instance_single_step_on_sled(cptestctx, nexus, original_sled, instance_id) + vmm_single_step_on_sled(cptestctx, nexus, original_sled, src_propolis_id) .await; - instance_single_step_on_sled(cptestctx, nexus, dst_sled_id, instance_id) + vmm_single_step_on_sled(cptestctx, nexus, dst_sled_id, dst_propolis_id) .await; instance_wait_for_state(&client, instance_id, InstanceState::Migrating) .await; @@ -1444,9 +1478,9 @@ async fn test_instance_metrics_with_migration( // Complete migration on the target. Simulated migrations always succeed. // After this the instance should be running and should continue to appear // to be provisioned. - instance_simulate_on_sled(cptestctx, nexus, original_sled, instance_id) + vmm_simulate_on_sled(cptestctx, nexus, original_sled, src_propolis_id) .await; - instance_simulate_on_sled(cptestctx, nexus, dst_sled_id, instance_id).await; + vmm_simulate_on_sled(cptestctx, nexus, dst_sled_id, dst_propolis_id).await; instance_wait_for_state(&client, instance_id, InstanceState::Running).await; check_provisioning_state(4, 1).await; @@ -3337,10 +3371,11 @@ async fn test_disks_detached_when_instance_destroyed( let apictx = &cptestctx.server.server_context(); let nexus = &apictx.nexus; let sa = nexus - .instance_sled_by_id(&instance_id) + .active_instance_info(&instance_id, None) .await .unwrap() - .expect("instance should be on a sled while it's running"); + .expect("instance should be on a sled while it's running") + .sled_client; // Stop and delete instance instance_post(&client, instance_name, InstanceOp::Stop).await; @@ -5080,28 +5115,29 @@ pub async fn assert_sled_vpc_routes( /// instance, and then tell it to finish simulating whatever async transition is /// going on. pub async fn instance_simulate(nexus: &Arc, id: &InstanceUuid) { - let sa = nexus - .instance_sled_by_id(id) + let sled_info = nexus + .active_instance_info(id, None) .await .unwrap() .expect("instance must be on a sled to simulate a state change"); - sa.instance_finish_transition(id.into_untyped_uuid()).await; + + sled_info.sled_client.vmm_finish_transition(sled_info.propolis_id).await; } /// Simulate one step of an ongoing instance state transition. To do this, we /// have to look up the instance, then get the sled agent associated with that /// instance, and then tell it to finish simulating whatever async transition is /// going on. -async fn instance_single_step_on_sled( +async fn vmm_single_step_on_sled( cptestctx: &ControlPlaneTestContext, nexus: &Arc, sled_id: SledUuid, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, ) { info!(&cptestctx.logctx.log, "Single-stepping simulated instance on sled"; - "instance_id" => %instance_id, "sled_id" => %sled_id); + "propolis_id" => %propolis_id, "sled_id" => %sled_id); let sa = nexus.sled_client(&sled_id).await.unwrap(); - sa.instance_single_step(instance_id.into_untyped_uuid()).await; + sa.vmm_single_step(propolis_id).await; } pub async fn instance_simulate_with_opctx( @@ -5109,27 +5145,28 @@ pub async fn instance_simulate_with_opctx( id: &InstanceUuid, opctx: &OpContext, ) { - let sa = nexus - .instance_sled_by_id_with_opctx(id, opctx) + let sled_info = nexus + .active_instance_info(id, Some(opctx)) .await .unwrap() .expect("instance must be on a sled to simulate a state change"); - sa.instance_finish_transition(id.into_untyped_uuid()).await; + + sled_info.sled_client.vmm_finish_transition(sled_info.propolis_id).await; } /// Simulates state transitions for the incarnation of the instance on the /// supplied sled (which may not be the sled ID currently stored in the /// instance's CRDB record). -async fn instance_simulate_on_sled( +async fn vmm_simulate_on_sled( cptestctx: &ControlPlaneTestContext, nexus: &Arc, sled_id: SledUuid, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, ) { info!(&cptestctx.logctx.log, "Poking simulated instance on sled"; - "instance_id" => %instance_id, "sled_id" => %sled_id); + "propolis_id" => %propolis_id, "sled_id" => %sled_id); let sa = nexus.sled_client(&sled_id).await.unwrap(); - sa.instance_finish_transition(instance_id.into_untyped_uuid()).await; + sa.vmm_finish_transition(propolis_id).await; } /// Simulates a migration source for the provided instance ID, sled ID, and @@ -5138,19 +5175,19 @@ async fn instance_simulate_migration_source( cptestctx: &ControlPlaneTestContext, nexus: &Arc, sled_id: SledUuid, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, migration_id: Uuid, ) { info!( &cptestctx.logctx.log, "Simulating migration source sled"; - "instance_id" => %instance_id, + "propolis_id" => %propolis_id, "sled_id" => %sled_id, "migration_id" => %migration_id, ); let sa = nexus.sled_client(&sled_id).await.unwrap(); - sa.instance_simulate_migration_source( - instance_id.into_untyped_uuid(), + sa.vmm_simulate_migration_source( + propolis_id, sled_agent_client::SimulateMigrationSource { migration_id, result: sled_agent_client::SimulatedMigrationResult::Success, diff --git a/nexus/tests/integration_tests/ip_pools.rs b/nexus/tests/integration_tests/ip_pools.rs index e872cc6fe3..f56755d85c 100644 --- a/nexus/tests/integration_tests/ip_pools.rs +++ b/nexus/tests/integration_tests/ip_pools.rs @@ -1344,12 +1344,12 @@ async fn test_ip_range_delete_with_allocated_external_ip_fails( .expect("Failed to stop instance"); // Simulate the transition, wait until it is in fact stopped. - let sa = nexus - .instance_sled_by_id(&instance_id) + let info = nexus + .active_instance_info(&instance_id, None) .await .unwrap() .expect("running instance should be on a sled"); - sa.instance_finish_transition(instance.identity.id).await; + info.sled_client.vmm_finish_transition(info.propolis_id).await; instance_wait_for_state(client, instance_id, InstanceState::Stopped).await; // Delete the instance diff --git a/nexus/tests/integration_tests/metrics.rs b/nexus/tests/integration_tests/metrics.rs index 9f4652c2da..f51f57d414 100644 --- a/nexus/tests/integration_tests/metrics.rs +++ b/nexus/tests/integration_tests/metrics.rs @@ -359,75 +359,12 @@ async fn test_instance_watcher_metrics( let nexus = &cptestctx.server.server_context().nexus; let oximeter = &cptestctx.oximeter; - // TODO(eliza): consider factoring this out to a generic - // `activate_background_task` function in `nexus-test-utils` eventually? let activate_instance_watcher = || async { - use nexus_client::types::BackgroundTask; - use nexus_client::types::CurrentStatus; - use nexus_client::types::CurrentStatusRunning; - use nexus_client::types::LastResult; - use nexus_client::types::LastResultCompleted; - - fn most_recent_start_time( - task: &BackgroundTask, - ) -> Option> { - match task.current { - CurrentStatus::Idle => match task.last { - LastResult::Completed(LastResultCompleted { - start_time, - .. - }) => Some(start_time), - LastResult::NeverCompleted => None, - }, - CurrentStatus::Running(CurrentStatusRunning { - start_time, - .. - }) => Some(start_time), - } - } + use nexus_test_utils::background::activate_background_task; + + let _ = activate_background_task(&internal_client, "instance_watcher") + .await; - eprintln!("\n --- activating instance watcher ---\n"); - let task = NexusRequest::object_get( - internal_client, - "/bgtasks/view/instance_watcher", - ) - .execute_and_parse_unwrap::() - .await; - let last_start = most_recent_start_time(&task); - - internal_client - .make_request( - http::Method::POST, - "/bgtasks/activate", - Some(serde_json::json!({ - "bgtask_names": vec![String::from("instance_watcher")] - })), - http::StatusCode::NO_CONTENT, - ) - .await - .unwrap(); - // Wait for the instance watcher task to finish - wait_for_condition( - || async { - let task = NexusRequest::object_get( - internal_client, - "/bgtasks/view/instance_watcher", - ) - .execute_and_parse_unwrap::() - .await; - if matches!(&task.current, CurrentStatus::Idle) - && most_recent_start_time(&task) > last_start - { - Ok(()) - } else { - Err(CondCheckError::<()>::NotYet) - } - }, - &Duration::from_millis(500), - &Duration::from_secs(60), - ) - .await - .unwrap(); // Make sure that the latest metrics have been collected. oximeter.force_collect().await; }; @@ -690,60 +627,106 @@ async fn test_mgs_metrics( return; } - let table = timeseries_query(&cptestctx, &format!("get {metric_name}")) - .await - .into_iter() - .find(|t| t.name() == metric_name); - let table = match table { - Some(table) => table, - None => panic!("missing table for {metric_name}"), + let query = format!("get {metric_name}"); + + // MGS polls SP sensor data once every second. It's possible that, when + // we triggered Oximeter to collect samples from MGS, it may not have + // run a poll yet, so retry this a few times to avoid a flaky failure if + // no simulated SPs have been polled yet. + // + // We really don't need to wait that long to know that the sensor + // metrics will never be present. This could probably be shorter + // than 30 seconds, but I want to be fairly generous to make sure + // there are no flaky failures even when things take way longer than + // expected... + const MAX_RETRY_DURATION: Duration = Duration::from_secs(30); + let result = wait_for_condition( + || async { + match check_inner(cptestctx, &metric_name, &query, &expected).await { + Ok(_) => Ok(()), + Err(e) => { + eprintln!("{e}; will try again to ensure all samples are collected"); + Err(CondCheckError::<()>::NotYet) + } + } + }, + &Duration::from_secs(1), + &MAX_RETRY_DURATION, + ) + .await; + if result.is_err() { + panic!( + "failed to find expected timeseries when running OxQL query \ + {query:?} within {MAX_RETRY_DURATION:?}" + ) }; - let mut found = expected - .keys() - .map(|serial| (serial.clone(), 0)) - .collect::>(); - for timeseries in table.timeseries() { - let fields = ×eries.fields; - let n_points = timeseries.points.len(); - assert!( - n_points > 0, - "{metric_name} timeseries {fields:?} should have points" - ); - let serial_str: &str = match timeseries.fields.get("chassis_serial") + // Note that *some* of these checks panic if they fail, but others call + // `anyhow::ensure!`. This is because, if we don't see all the expected + // timeseries, it's possible that this is because some sensor polls + // haven't completed yet, so we'll retry those checks a few times. On + // the other hand, if we see malformed timeseries, or timeseries that we + // don't expect to exist, that means something has gone wrong, and we + // will fail the test immediately. + async fn check_inner( + cptestctx: &ControlPlaneTestContext, + name: &str, + query: &str, + expected: &HashMap, + ) -> anyhow::Result<()> { + cptestctx.oximeter.force_collect().await; + let table = timeseries_query(&cptestctx, &query) + .await + .into_iter() + .find(|t| t.name() == name) + .ok_or_else(|| { + anyhow::anyhow!("failed to find table for {query}") + })?; + + let mut found = expected + .keys() + .map(|serial| (serial.clone(), 0)) + .collect::>(); + for timeseries in table.timeseries() { + let fields = ×eries.fields; + let n_points = timeseries.points.len(); + anyhow::ensure!( + n_points > 0, + "{name} timeseries {fields:?} should have points" + ); + let serial_str: &str = match timeseries.fields.get("chassis_serial") { Some(FieldValue::String(s)) => s.borrow(), Some(x) => panic!( - "{metric_name} `chassis_serial` field should be a string, but got: {x:?}" + "{name} `chassis_serial` field should be a string, but got: {x:?}" ), None => { - panic!("{metric_name} timeseries should have a `chassis_serial` field") + panic!("{name} timeseries should have a `chassis_serial` field") } }; - if let Some(count) = found.get_mut(serial_str) { - *count += 1; - } else { - panic!( - "{metric_name} timeseries had an unexpected chassis serial \ - number {serial_str:?} (not in the config file)", - ); + if let Some(count) = found.get_mut(serial_str) { + *count += 1; + } else { + panic!( + "{name} timeseries had an unexpected chassis serial \ + number {serial_str:?} (not in the config file)", + ); + } } - } - eprintln!("-> {metric_name}: found timeseries: {found:#?}"); - assert_eq!( - found, expected, - "number of {metric_name} timeseries didn't match expected in {table:#?}", - ); - eprintln!("-> okay, looks good!"); + eprintln!("-> {name}: found timeseries: {found:#?}"); + anyhow::ensure!( + &found == expected, + "number of {name} timeseries didn't match expected in {table:#?}", + ); + eprintln!("-> okay, looks good!"); + Ok(()) + } } // Wait until the MGS registers as a producer with Oximeter. wait_for_producer(&cptestctx.oximeter, &mgs.gateway_id).await; - // ...and collect its samples. - cptestctx.oximeter.force_collect().await; - check_all_timeseries_present(&cptestctx, "temperature", temp_sensors).await; check_all_timeseries_present(&cptestctx, "voltage", voltage_sensors).await; check_all_timeseries_present(&cptestctx, "current", current_sensors).await; diff --git a/nexus/tests/integration_tests/pantry.rs b/nexus/tests/integration_tests/pantry.rs index d77ad49db6..22d35b01b5 100644 --- a/nexus/tests/integration_tests/pantry.rs +++ b/nexus/tests/integration_tests/pantry.rs @@ -88,12 +88,12 @@ async fn set_instance_state( } async fn instance_simulate(nexus: &Arc, id: &InstanceUuid) { - let sa = nexus - .instance_sled_by_id(id) + let info = nexus + .active_instance_info(id, None) .await .unwrap() .expect("instance must be on a sled to simulate a state change"); - sa.instance_finish_transition(id.into_untyped_uuid()).await; + info.sled_client.vmm_finish_transition(info.propolis_id).await; } async fn disk_get(client: &ClientTestContext, disk_url: &str) -> Disk { diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index bf73855ea7..5201b5c971 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -954,12 +954,12 @@ async fn dbinit_equals_sum_of_all_up() { // Create a connection pool after we apply the first schema version but // before applying the rest, and grab a connection from that pool. We'll use // it for an extra check later. - let pool = nexus_db_queries::db::Pool::new( + let pool = nexus_db_queries::db::Pool::new_single_host( log, &nexus_db_queries::db::Config { url: crdb.pg_config().clone() }, ); let conn_from_pool = - pool.pool().get().await.expect("failed to get pooled connection"); + pool.claim().await.expect("failed to get pooled connection"); // Go from the second version to the latest version. for version in all_versions.iter_versions().skip(1) { diff --git a/nexus/tests/output/cmd-nexus-noargs-stderr b/nexus/tests/output/cmd-nexus-noargs-stderr index 385248bd0e..5a218b5c94 100644 --- a/nexus/tests/output/cmd-nexus-noargs-stderr +++ b/nexus/tests/output/cmd-nexus-noargs-stderr @@ -1,12 +1,11 @@ See README.adoc for more information -Usage: nexus [OPTIONS] [CONFIG_FILE_PATH] +Usage: nexus [CONFIG_FILE_PATH] Arguments: [CONFIG_FILE_PATH] Options: - -O, --openapi Print the external OpenAPI Spec document and exit - -h, --help Print help + -h, --help Print help nexus: CONFIG_FILE_PATH is required diff --git a/nexus/tests/output/cmd-nexus-openapi-stderr b/nexus/tests/output/cmd-nexus-openapi-stderr deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index fd5ef251dd..8af94fd25e 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -10,10 +10,10 @@ workspace = true [dependencies] anyhow.workspace = true async-trait.workspace = true +base64.workspace = true chrono.workspace = true clap.workspace = true cookie.workspace = true -base64.workspace = true derive-where.workspace = true derive_more.workspace = true dropshot.workspace = true diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index dcc96976ef..7d4d9f72c5 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -544,7 +544,7 @@ impl BlueprintZonesConfig { } /// Returns true if all zones in the blueprint have a disposition of - // `Expunged`, false otherwise. + /// `Expunged`, false otherwise. pub fn are_all_zones_expunged(&self) -> bool { self.zones .iter() diff --git a/nexus/types/src/deployment/planning_input.rs b/nexus/types/src/deployment/planning_input.rs index dabb47066e..04a85bc179 100644 --- a/nexus/types/src/deployment/planning_input.rs +++ b/nexus/types/src/deployment/planning_input.rs @@ -95,6 +95,10 @@ impl PlanningInput { self.policy.target_nexus_zone_count } + pub fn target_internal_dns_zone_count(&self) -> usize { + self.policy.target_internal_dns_zone_count + } + pub fn target_cockroachdb_zone_count(&self) -> usize { self.policy.target_cockroachdb_zone_count } @@ -709,6 +713,11 @@ pub struct Policy { /// desired total number of deployed Nexus zones pub target_nexus_zone_count: usize, + /// desired total number of internal DNS zones. + /// Must be <= [`omicron_common::policy::MAX_INTERNAL_DNS_REDUNDANCY`], + /// and should be >= [`omicron_common::policy::INTERNAL_DNS_REDUNDANCY`]. + pub target_internal_dns_zone_count: usize, + /// desired total number of deployed CockroachDB zones pub target_cockroachdb_zone_count: usize, @@ -782,6 +791,7 @@ impl PlanningInputBuilder { service_ip_pool_ranges: Vec::new(), target_boundary_ntp_zone_count: 0, target_nexus_zone_count: 0, + target_internal_dns_zone_count: 0, target_cockroachdb_zone_count: 0, target_cockroachdb_cluster_version: CockroachDbClusterVersion::POLICY, diff --git a/nexus/types/src/deployment/zone_type.rs b/nexus/types/src/deployment/zone_type.rs index e4958fc3c3..e0f389fe2a 100644 --- a/nexus/types/src/deployment/zone_type.rs +++ b/nexus/types/src/deployment/zone_type.rs @@ -73,56 +73,26 @@ impl BlueprintZoneType { /// Identifies whether this is an NTP zone (any flavor) pub fn is_ntp(&self) -> bool { - match self { + matches!( + self, BlueprintZoneType::InternalNtp(_) - | BlueprintZoneType::BoundaryNtp(_) => true, - BlueprintZoneType::Nexus(_) - | BlueprintZoneType::ExternalDns(_) - | BlueprintZoneType::Clickhouse(_) - | BlueprintZoneType::ClickhouseKeeper(_) - | BlueprintZoneType::ClickhouseServer(_) - | BlueprintZoneType::CockroachDb(_) - | BlueprintZoneType::Crucible(_) - | BlueprintZoneType::CruciblePantry(_) - | BlueprintZoneType::InternalDns(_) - | BlueprintZoneType::Oximeter(_) => false, - } + | BlueprintZoneType::BoundaryNtp(_) + ) } /// Identifies whether this is a Nexus zone pub fn is_nexus(&self) -> bool { - match self { - BlueprintZoneType::Nexus(_) => true, - BlueprintZoneType::BoundaryNtp(_) - | BlueprintZoneType::ExternalDns(_) - | BlueprintZoneType::Clickhouse(_) - | BlueprintZoneType::ClickhouseKeeper(_) - | BlueprintZoneType::ClickhouseServer(_) - | BlueprintZoneType::CockroachDb(_) - | BlueprintZoneType::Crucible(_) - | BlueprintZoneType::CruciblePantry(_) - | BlueprintZoneType::InternalDns(_) - | BlueprintZoneType::InternalNtp(_) - | BlueprintZoneType::Oximeter(_) => false, - } + matches!(self, BlueprintZoneType::Nexus(_)) + } + + /// Identifies whether this is an internal DNS zone + pub fn is_internal_dns(&self) -> bool { + matches!(self, BlueprintZoneType::InternalDns(_)) } /// Identifies whether this a Crucible (not Crucible pantry) zone pub fn is_crucible(&self) -> bool { - match self { - BlueprintZoneType::Crucible(_) => true, - BlueprintZoneType::BoundaryNtp(_) - | BlueprintZoneType::Clickhouse(_) - | BlueprintZoneType::ClickhouseKeeper(_) - | BlueprintZoneType::ClickhouseServer(_) - | BlueprintZoneType::CockroachDb(_) - | BlueprintZoneType::CruciblePantry(_) - | BlueprintZoneType::ExternalDns(_) - | BlueprintZoneType::InternalDns(_) - | BlueprintZoneType::InternalNtp(_) - | BlueprintZoneType::Nexus(_) - | BlueprintZoneType::Oximeter(_) => false, - } + matches!(self, BlueprintZoneType::Crucible(_)) } /// Returns the durable dataset associated with this zone, if any exists. diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 83897cbd1d..691f36534d 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -8,6 +8,7 @@ use crate::external_api::shared; use base64::Engine; use chrono::{DateTime, Utc}; +use http::Uri; use omicron_common::api::external::{ AddressLotKind, AllowedSourceIps, BfdMode, BgpPeer, ByteCount, Hostname, IdentityMetadataCreateParams, IdentityMetadataUpdateParams, @@ -16,6 +17,7 @@ use omicron_common::api::external::{ }; use omicron_common::disk::DiskVariant; use oxnet::{IpNet, Ipv4Net, Ipv6Net}; +use parse_display::Display; use schemars::JsonSchema; use serde::{ de::{self, Visitor}, @@ -83,11 +85,13 @@ path_param!(IpPoolPath, pool, "IP pool"); path_param!(SshKeyPath, ssh_key, "SSH key"); path_param!(AddressLotPath, address_lot, "address lot"); path_param!(ProbePath, probe, "probe"); +path_param!(CertificatePath, certificate, "certificate"); id_path_param!(GroupPath, group_id, "group"); // TODO: The hardware resources should be represented by its UUID or a hardware // ID that can be used to deterministically generate the UUID. +id_path_param!(RackPath, rack_id, "rack"); id_path_param!(SledPath, sled_id, "sled"); id_path_param!(SwitchPath, switch_id, "switch"); id_path_param!(PhysicalDiskPath, disk_id, "physical disk"); @@ -141,6 +145,13 @@ pub struct OptionalSiloSelector { pub silo: Option, } +/// Path parameters for Silo User requests +#[derive(Deserialize, JsonSchema)] +pub struct UserParam { + /// The user's internal ID + pub user_id: Uuid, +} + #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] pub struct SamlIdentityProviderSelector { /// Name or ID of the silo in which the SAML identity provider is associated @@ -1241,6 +1252,24 @@ pub struct RouterRouteUpdate { // DISKS +#[derive(Display, Serialize, Deserialize, JsonSchema)] +#[display(style = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum DiskMetricName { + Activated, + Flush, + Read, + ReadBytes, + Write, + WriteBytes, +} + +#[derive(Serialize, Deserialize, JsonSchema)] +pub struct DiskMetricsPath { + pub disk: NameOrId, + pub metric: DiskMetricName, +} + #[derive(Copy, Clone, Debug, Deserialize, Serialize)] #[serde(try_from = "u32")] // invoke the try_from validation routine below pub struct BlockSize(pub u32); @@ -1421,6 +1450,23 @@ pub struct LoopbackAddressCreate { pub anycast: bool, } +#[derive(Serialize, Deserialize, JsonSchema)] +pub struct LoopbackAddressPath { + /// The rack to use when selecting the loopback address. + pub rack_id: Uuid, + + /// The switch location to use when selecting the loopback address. + pub switch_location: Name, + + /// The IP address and subnet mask to use when selecting the loopback + /// address. + pub address: IpAddr, + + /// The IP address and subnet mask to use when selecting the loopback + /// address. + pub subnet_mask: u8, +} + /// Parameters for creating a port settings group. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct SwtichPortSettingsGroupCreate { @@ -1897,6 +1943,20 @@ pub struct SshKeyCreate { // METRICS +#[derive(Display, Deserialize, JsonSchema)] +#[display(style = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum SystemMetricName { + VirtualDiskSpaceProvisioned, + CpusProvisioned, + RamProvisioned, +} + +#[derive(Deserialize, JsonSchema)] +pub struct SystemMetricsPathParam { + pub metric_name: SystemMetricName, +} + /// Query parameters common to resource metrics endpoints. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct ResourceMetrics { @@ -1958,3 +2018,98 @@ pub struct AllowListUpdate { /// The new list of allowed source IPs. pub allowed_ips: AllowedSourceIps, } + +// Roles + +// Roles have their own pagination scheme because they do not use the usual "id" +// or "name" types. For more, see the comment in dbinit.sql. +#[derive(Deserialize, JsonSchema, Serialize)] +pub struct RolePage { + pub last_seen: String, +} + +/// Path parameters for global (system) role requests +#[derive(Deserialize, JsonSchema)] +pub struct RolePath { + /// The built-in role's unique name. + pub role_name: String, +} + +// Console API + +#[derive(Deserialize, JsonSchema)] +pub struct RestPathParam { + pub path: Vec, +} + +#[derive(Deserialize, JsonSchema)] +pub struct LoginToProviderPathParam { + pub silo_name: Name, + pub provider_name: Name, +} + +#[derive(Serialize, Deserialize, JsonSchema)] +pub struct LoginUrlQuery { + pub redirect_uri: Option, +} + +#[derive(Deserialize, JsonSchema)] +pub struct LoginPath { + pub silo_name: Name, +} + +/// This is meant as a security feature. We want to ensure we never redirect to +/// a URI on a different host. +#[derive(Serialize, Deserialize, Debug, JsonSchema, Clone, Display)] +#[serde(try_from = "String")] +#[display("{0}")] +pub struct RelativeUri(String); + +impl FromStr for RelativeUri { + type Err = String; + + fn from_str(s: &str) -> Result { + Self::try_from(s.to_string()) + } +} + +impl TryFrom for RelativeUri { + type Error = String; + + fn try_from(uri: Uri) -> Result { + if uri.host().is_none() && uri.scheme().is_none() { + Ok(Self(uri.to_string())) + } else { + Err(format!("\"{}\" is not a relative URI", uri)) + } + } +} + +impl TryFrom for RelativeUri { + type Error = String; + + fn try_from(s: String) -> Result { + s.parse::() + .map_err(|_| format!("\"{}\" is not a relative URI", s)) + .and_then(|uri| Self::try_from(uri)) + } +} + +// Device auth + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct DeviceAuthRequest { + pub client_id: Uuid, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct DeviceAuthVerify { + pub user_code: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct DeviceAccessTokenRequest { + pub grant_type: String, + pub device_code: String, + pub client_id: Uuid, +} diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 58c2e560ab..e8d81b05bb 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -498,6 +498,12 @@ pub struct Rack { // SLEDS +/// The unique ID of a sled. +#[derive(Clone, Debug, Serialize, JsonSchema)] +pub struct SledId { + pub id: Uuid, +} + /// An operator's view of a Sled. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct Sled { diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 111bd552d0..da8bbacf8b 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -746,44 +746,6 @@ } } }, - "/instances/{instance_id}": { - "put": { - "summary": "Report updated state for an instance.", - "operationId": "cpapi_instances_put", - "parameters": [ - { - "in": "path", - "name": "instance_id", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SledInstanceState" - } - } - }, - "required": true - }, - "responses": { - "204": { - "description": "resource updated" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, "/instances/{instance_id}/migrate": { "post": { "operationId": "instance_migrate", @@ -1470,6 +1432,43 @@ } } }, + "/vmms/{propolis_id}": { + "put": { + "summary": "Report updated state for a VMM.", + "operationId": "cpapi_instances_put", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForPropolisKind" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledVmmState" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/volume/{volume_id}/remove-read-only-parent": { "post": { "summary": "Request removal of a read_only_parent from a volume.", @@ -2712,39 +2711,8 @@ ] }, "DatasetKind": { - "description": "Describes the purpose of the dataset.", - "oneOf": [ - { - "type": "string", - "enum": [ - "crucible", - "cockroach", - "external_dns", - "internal_dns" - ] - }, - { - "description": "Used for single-node clickhouse deployments", - "type": "string", - "enum": [ - "clickhouse" - ] - }, - { - "description": "Used for replicated clickhouse deployments", - "type": "string", - "enum": [ - "clickhouse_keeper" - ] - }, - { - "description": "Used for replicated clickhouse deployments", - "type": "string", - "enum": [ - "clickhouse_server" - ] - } - ] + "description": "The kind of dataset. See the `DatasetKind` enum in omicron-common for possible values.", + "type": "string" }, "DatasetPutRequest": { "description": "Describes a dataset within a pool.", @@ -5062,50 +5030,6 @@ "id" ] }, - "SledInstanceState": { - "description": "A wrapper type containing a sled's total knowledge of the state of a specific VMM and the instance it incarnates.", - "type": "object", - "properties": { - "migration_in": { - "nullable": true, - "description": "The current state of any inbound migration to this VMM.", - "allOf": [ - { - "$ref": "#/components/schemas/MigrationRuntimeState" - } - ] - }, - "migration_out": { - "nullable": true, - "description": "The state of any outbound migration from this VMM.", - "allOf": [ - { - "$ref": "#/components/schemas/MigrationRuntimeState" - } - ] - }, - "propolis_id": { - "description": "The ID of the VMM whose state is being reported.", - "allOf": [ - { - "$ref": "#/components/schemas/TypedUuidForPropolisKind" - } - ] - }, - "vmm_state": { - "description": "The most recent state of the sled's VMM process.", - "allOf": [ - { - "$ref": "#/components/schemas/VmmRuntimeState" - } - ] - } - }, - "required": [ - "propolis_id", - "vmm_state" - ] - }, "SledPolicy": { "description": "The operator-defined policy of a sled.", "oneOf": [ @@ -5220,6 +5144,41 @@ } ] }, + "SledVmmState": { + "description": "A wrapper type containing a sled's total knowledge of the state of a VMM.", + "type": "object", + "properties": { + "migration_in": { + "nullable": true, + "description": "The current state of any inbound migration to this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/MigrationRuntimeState" + } + ] + }, + "migration_out": { + "nullable": true, + "description": "The state of any outbound migration from this VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/MigrationRuntimeState" + } + ] + }, + "vmm_state": { + "description": "The most recent state of the sled's VMM process.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmRuntimeState" + } + ] + } + }, + "required": [ + "vmm_state" + ] + }, "SourceNatConfig": { "description": "An IP address and port range used for source NAT, i.e., making outbound network connections from guests or services.", "type": "object", @@ -5332,10 +5291,6 @@ "type": "string", "format": "uuid" }, - "TypedUuidForPropolisKind": { - "type": "string", - "format": "uuid" - }, "TypedUuidForSledKind": { "type": "string", "format": "uuid" @@ -5597,6 +5552,10 @@ ] } ] + }, + "TypedUuidForPropolisKind": { + "type": "string", + "format": "uuid" } }, "responses": { diff --git a/openapi/nexus.json b/openapi/nexus.json index 47f1f0822b..a855378cd4 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -468,6 +468,7 @@ { "in": "path", "name": "certificate", + "description": "Name or ID of the certificate", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -504,6 +505,7 @@ { "in": "path", "name": "certificate", + "description": "Name or ID of the certificate", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -4167,7 +4169,7 @@ { "in": "path", "name": "rack_id", - "description": "The rack's unique ID.", + "description": "ID of the rack", "required": true, "schema": { "type": "string", @@ -5028,7 +5030,7 @@ { "in": "path", "name": "user_id", - "description": "The user's internal id", + "description": "The user's internal ID", "required": true, "schema": { "type": "string", @@ -5070,7 +5072,7 @@ { "in": "path", "name": "user_id", - "description": "The user's internal id", + "description": "The user's internal ID", "required": true, "schema": { "type": "string", @@ -7872,7 +7874,7 @@ { "in": "path", "name": "user_id", - "description": "The user's internal id", + "description": "The user's internal ID", "required": true, "schema": { "type": "string", diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 4c40fb5da0..bb8e4e0b87 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -176,37 +176,17 @@ } } }, - "/disks/{disk_id}": { - "put": { - "operationId": "disk_put", - "parameters": [ - { - "in": "path", - "name": "disk_id", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DiskEnsureBody" - } - } - }, - "required": true - }, + "/datasets": { + "get": { + "summary": "Lists the datasets that this sled is configured to use", + "operationId": "datasets_get", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/DiskRuntimeState" + "$ref": "#/components/schemas/DatasetsConfig" } } } @@ -218,26 +198,15 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/instances/{instance_id}": { + }, "put": { - "operationId": "instance_register", - "parameters": [ - { - "in": "path", - "name": "instance_id", - "required": true, - "schema": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" - } - } - ], + "summary": "Configures datasets to be used on this sled", + "operationId": "datasets_put", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceEnsureBody" + "$ref": "#/components/schemas/DatasetsConfig" } } }, @@ -249,38 +218,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SledInstanceState" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "operationId": "instance_unregister", - "parameters": [ - { - "in": "path", - "name": "instance_id", - "required": true, - "schema": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceUnregisterResponse" + "$ref": "#/components/schemas/DatasetsManagementResult" } } } @@ -294,10 +232,9 @@ } } }, - "/instances/{instance_id}/disks/{disk_id}/snapshot": { - "post": { - "summary": "Take a snapshot of a disk that is attached to an instance", - "operationId": "instance_issue_disk_snapshot_request", + "/disks/{disk_id}": { + "put": { + "operationId": "disk_put", "parameters": [ { "in": "path", @@ -307,166 +244,13 @@ "type": "string", "format": "uuid" } - }, - { - "in": "path", - "name": "instance_id", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - } } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceIssueDiskSnapshotRequestBody" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceIssueDiskSnapshotRequestResponse" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/instances/{instance_id}/external-ip": { - "put": { - "operationId": "instance_put_external_ip", - "parameters": [ - { - "in": "path", - "name": "instance_id", - "required": true, - "schema": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceExternalIpBody" - } - } - }, - "required": true - }, - "responses": { - "204": { - "description": "resource updated" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "operationId": "instance_delete_external_ip", - "parameters": [ - { - "in": "path", - "name": "instance_id", - "required": true, - "schema": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceExternalIpBody" - } - } - }, - "required": true - }, - "responses": { - "204": { - "description": "resource updated" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/instances/{instance_id}/state": { - "get": { - "operationId": "instance_get_state", - "parameters": [ - { - "in": "path", - "name": "instance_id", - "required": true, - "schema": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SledInstanceState" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "operationId": "instance_put_state", - "parameters": [ - { - "in": "path", - "name": "instance_id", - "required": true, - "schema": { - "$ref": "#/components/schemas/TypedUuidForInstanceKind" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstancePutStateBody" + "$ref": "#/components/schemas/DiskEnsureBody" } } }, @@ -478,7 +262,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstancePutStateResponse" + "$ref": "#/components/schemas/DiskRuntimeState" } } } @@ -853,12 +637,267 @@ } } }, - "required": true - }, - "responses": { - "204": { - "description": "resource updated" - }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "summary": "Delete a mapping from a virtual NIC to a physical host", + "operationId": "del_v2p", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VirtualNetworkInterfaceHost" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}": { + "put": { + "operationId": "vmm_register", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForPropolisKind" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceEnsureBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledVmmState" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "vmm_unregister", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForPropolisKind" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmUnregisterResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/disks/{disk_id}/snapshot": { + "post": { + "summary": "Take a snapshot of a disk that is attached to an instance", + "operationId": "vmm_issue_disk_snapshot_request", + "parameters": [ + { + "in": "path", + "name": "disk_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForPropolisKind" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmIssueDiskSnapshotRequestBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmIssueDiskSnapshotRequestResponse" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/external-ip": { + "put": { + "operationId": "vmm_put_external_ip", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForPropolisKind" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceExternalIpBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "operationId": "vmm_delete_external_ip", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForPropolisKind" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceExternalIpBody" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/vmms/{propolis_id}/state": { + "get": { + "operationId": "vmm_get_state", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForPropolisKind" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledVmmState" + } + } + } + }, "4XX": { "$ref": "#/components/responses/Error" }, @@ -867,22 +906,38 @@ } } }, - "delete": { - "summary": "Delete a mapping from a virtual NIC to a physical host", - "operationId": "del_v2p", + "put": { + "operationId": "vmm_put_state", + "parameters": [ + { + "in": "path", + "name": "propolis_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/TypedUuidForPropolisKind" + } + } + ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VirtualNetworkInterfaceHost" + "$ref": "#/components/schemas/VmmPutStateBody" } } }, "required": true }, "responses": { - "204": { - "description": "resource updated" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VmmPutStateResponse" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -1980,83 +2035,311 @@ "description": "The count of bundles / bytes removed during a cleanup operation.", "type": "object", "properties": { - "bundles": { - "description": "The number of bundles removed.", - "type": "integer", - "format": "uint64", - "minimum": 0 + "bundles": { + "description": "The number of bundles removed.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "bytes": { + "description": "The number of bytes removed.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "bundles", + "bytes" + ] + }, + "CleanupPeriod": { + "description": "A period on which bundles are automatically cleaned up.", + "allOf": [ + { + "$ref": "#/components/schemas/Duration" + } + ] + }, + "CompressionAlgorithm": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "on" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "off" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "gzip" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "level": { + "$ref": "#/components/schemas/GzipLevel" + }, + "type": { + "type": "string", + "enum": [ + "gzip_n" + ] + } + }, + "required": [ + "level", + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lz4" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "lzjb" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "zle" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, + "CrucibleOpts": { + "description": "CrucibleOpts\n\n
JSON schema\n\n```json { \"type\": \"object\", \"required\": [ \"id\", \"lossy\", \"read_only\", \"target\" ], \"properties\": { \"cert_pem\": { \"type\": [ \"string\", \"null\" ] }, \"control\": { \"type\": [ \"string\", \"null\" ] }, \"flush_timeout\": { \"type\": [ \"number\", \"null\" ], \"format\": \"float\" }, \"id\": { \"type\": \"string\", \"format\": \"uuid\" }, \"key\": { \"type\": [ \"string\", \"null\" ] }, \"key_pem\": { \"type\": [ \"string\", \"null\" ] }, \"lossy\": { \"type\": \"boolean\" }, \"read_only\": { \"type\": \"boolean\" }, \"root_cert_pem\": { \"type\": [ \"string\", \"null\" ] }, \"target\": { \"type\": \"array\", \"items\": { \"type\": \"string\" } } } } ```
", + "type": "object", + "properties": { + "cert_pem": { + "nullable": true, + "type": "string" + }, + "control": { + "nullable": true, + "type": "string" + }, + "flush_timeout": { + "nullable": true, + "type": "number", + "format": "float" + }, + "id": { + "type": "string", + "format": "uuid" + }, + "key": { + "nullable": true, + "type": "string" + }, + "key_pem": { + "nullable": true, + "type": "string" + }, + "lossy": { + "type": "boolean" + }, + "read_only": { + "type": "boolean" + }, + "root_cert_pem": { + "nullable": true, + "type": "string" + }, + "target": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "id", + "lossy", + "read_only", + "target" + ] + }, + "DatasetConfig": { + "description": "Configuration information necessary to request a single dataset", + "type": "object", + "properties": { + "compression": { + "description": "The compression mode to be used by the dataset", + "allOf": [ + { + "$ref": "#/components/schemas/CompressionAlgorithm" + } + ] + }, + "id": { + "description": "The UUID of the dataset being requested", + "allOf": [ + { + "$ref": "#/components/schemas/TypedUuidForDatasetKind" + } + ] + }, + "name": { + "description": "The dataset's name", + "allOf": [ + { + "$ref": "#/components/schemas/DatasetName" + } + ] + }, + "quota": { + "nullable": true, + "description": "The upper bound on the amount of storage used by this dataset", + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "reservation": { + "nullable": true, + "description": "The lower bound on the amount of storage usable by this dataset", + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "compression", + "id", + "name" + ] + }, + "DatasetKind": { + "description": "The kind of dataset. See the `DatasetKind` enum in omicron-common for possible values.", + "type": "string" + }, + "DatasetManagementStatus": { + "description": "Identifies how a single dataset management operation may have succeeded or failed.", + "type": "object", + "properties": { + "dataset_name": { + "$ref": "#/components/schemas/DatasetName" + }, + "err": { + "nullable": true, + "type": "string" + } + }, + "required": [ + "dataset_name" + ] + }, + "DatasetName": { + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/DatasetKind" }, - "bytes": { - "description": "The number of bytes removed.", - "type": "integer", - "format": "uint64", - "minimum": 0 + "pool_name": { + "$ref": "#/components/schemas/ZpoolName" } }, "required": [ - "bundles", - "bytes" + "kind", + "pool_name" ] }, - "CleanupPeriod": { - "description": "A period on which bundles are automatically cleaned up.", - "allOf": [ - { - "$ref": "#/components/schemas/Duration" + "DatasetsConfig": { + "type": "object", + "properties": { + "datasets": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/DatasetConfig" + } + }, + "generation": { + "description": "generation number of this configuration\n\nThis generation number is owned by the control plane (i.e., RSS or Nexus, depending on whether RSS-to-Nexus handoff has happened). It should not be bumped within Sled Agent.\n\nSled Agent rejects attempts to set the configuration to a generation older than the one it's currently running.\n\nNote that \"Generation::new()\", AKA, the first generation number, is reserved for \"no datasets\". This is the default configuration for a sled before any requests have been made.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] } + }, + "required": [ + "datasets", + "generation" ] }, - "CrucibleOpts": { - "description": "CrucibleOpts\n\n
JSON schema\n\n```json { \"type\": \"object\", \"required\": [ \"id\", \"lossy\", \"read_only\", \"target\" ], \"properties\": { \"cert_pem\": { \"type\": [ \"string\", \"null\" ] }, \"control\": { \"type\": [ \"string\", \"null\" ] }, \"flush_timeout\": { \"type\": [ \"number\", \"null\" ], \"format\": \"float\" }, \"id\": { \"type\": \"string\", \"format\": \"uuid\" }, \"key\": { \"type\": [ \"string\", \"null\" ] }, \"key_pem\": { \"type\": [ \"string\", \"null\" ] }, \"lossy\": { \"type\": \"boolean\" }, \"read_only\": { \"type\": \"boolean\" }, \"root_cert_pem\": { \"type\": [ \"string\", \"null\" ] }, \"target\": { \"type\": \"array\", \"items\": { \"type\": \"string\" } } } } ```
", + "DatasetsManagementResult": { + "description": "The result from attempting to manage datasets.", "type": "object", "properties": { - "cert_pem": { - "nullable": true, - "type": "string" - }, - "control": { - "nullable": true, - "type": "string" - }, - "flush_timeout": { - "nullable": true, - "type": "number", - "format": "float" - }, - "id": { - "type": "string", - "format": "uuid" - }, - "key": { - "nullable": true, - "type": "string" - }, - "key_pem": { - "nullable": true, - "type": "string" - }, - "lossy": { - "type": "boolean" - }, - "read_only": { - "type": "boolean" - }, - "root_cert_pem": { - "nullable": true, - "type": "string" - }, - "target": { + "status": { "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/DatasetManagementStatus" } } }, "required": [ - "id", - "lossy", - "read_only", - "target" + "status" ] }, "DhcpConfig": { @@ -2701,6 +2984,11 @@ "format": "uint64", "minimum": 0 }, + "GzipLevel": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, "HostIdentifier": { "description": "A `HostIdentifier` represents either an IP host or network (v4 or v6), or an entire VPC (identified by its VNI). It is used in firewall rule host filters.", "oneOf": [ @@ -2837,6 +3125,14 @@ } ] }, + "instance_id": { + "description": "The ID of the instance for which this VMM is being created.", + "allOf": [ + { + "$ref": "#/components/schemas/TypedUuidForInstanceKind" + } + ] + }, "instance_runtime": { "description": "The instance runtime state for the instance being registered.", "allOf": [ @@ -2857,14 +3153,6 @@ "description": "The address at which this VMM should serve a Propolis server API.", "type": "string" }, - "propolis_id": { - "description": "The ID of the VMM being registered. This may not be the active VMM ID in the instance runtime state (e.g. if the new VMM is going to be a migration target).", - "allOf": [ - { - "$ref": "#/components/schemas/TypedUuidForPropolisKind" - } - ] - }, "vmm_runtime": { "description": "The initial VMM runtime state for the VMM being registered.", "allOf": [ @@ -2876,10 +3164,10 @@ }, "required": [ "hardware", + "instance_id", "instance_runtime", "metadata", "propolis_addr", - "propolis_id", "vmm_runtime" ] }, @@ -2985,30 +3273,6 @@ "source_nat" ] }, - "InstanceIssueDiskSnapshotRequestBody": { - "type": "object", - "properties": { - "snapshot_id": { - "type": "string", - "format": "uuid" - } - }, - "required": [ - "snapshot_id" - ] - }, - "InstanceIssueDiskSnapshotRequestResponse": { - "type": "object", - "properties": { - "snapshot_id": { - "type": "string", - "format": "uuid" - } - }, - "required": [ - "snapshot_id" - ] - }, "InstanceMetadata": { "description": "Metadata used to track statistics about an instance.", "type": "object", @@ -3058,181 +3322,65 @@ } ] }, - "memory": { - "$ref": "#/components/schemas/ByteCount" - }, - "ncpus": { - "$ref": "#/components/schemas/InstanceCpuCount" - } - }, - "required": [ - "hostname", - "memory", - "ncpus" - ] - }, - "InstancePutStateBody": { - "description": "The body of a request to move a previously-ensured instance into a specific runtime state.", - "type": "object", - "properties": { - "state": { - "description": "The state into which the instance should be driven.", - "allOf": [ - { - "$ref": "#/components/schemas/InstanceStateRequested" - } - ] - } - }, - "required": [ - "state" - ] - }, - "InstancePutStateResponse": { - "description": "The response sent from a request to move an instance into a specific runtime state.", - "type": "object", - "properties": { - "updated_runtime": { - "nullable": true, - "description": "The current runtime state of the instance after handling the request to change its state. If the instance's state did not change, this field is `None`.", - "allOf": [ - { - "$ref": "#/components/schemas/SledInstanceState" - } - ] - } - } - }, - "InstanceRuntimeState": { - "description": "The dynamic runtime properties of an instance: its current VMM ID (if any), migration information (if any), and the instance state to report if there is no active VMM.", - "type": "object", - "properties": { - "dst_propolis_id": { - "nullable": true, - "description": "If a migration is active, the ID of the target VMM.", - "allOf": [ - { - "$ref": "#/components/schemas/TypedUuidForPropolisKind" - } - ] - }, - "gen": { - "description": "Generation number for this state.", - "allOf": [ - { - "$ref": "#/components/schemas/Generation" - } - ] - }, - "migration_id": { - "nullable": true, - "description": "If a migration is active, the ID of that migration.", - "type": "string", - "format": "uuid" - }, - "propolis_id": { - "nullable": true, - "description": "The instance's currently active VMM ID.", - "allOf": [ - { - "$ref": "#/components/schemas/TypedUuidForPropolisKind" - } - ] - }, - "time_updated": { - "description": "Timestamp for this information.", - "type": "string", - "format": "date-time" - } - }, - "required": [ - "gen", - "time_updated" - ] - }, - "InstanceStateRequested": { - "description": "Requestable running state of an Instance.\n\nA subset of [`omicron_common::api::external::InstanceState`].", - "oneOf": [ - { - "description": "Run this instance by migrating in from a previous running incarnation of the instance.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "migration_target" - ] - }, - "value": { - "$ref": "#/components/schemas/InstanceMigrationTargetParams" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Start the instance if it is not already running.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "running" - ] - } - }, - "required": [ - "type" - ] - }, - { - "description": "Stop the instance.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "stopped" - ] - } - }, - "required": [ - "type" - ] - }, - { - "description": "Immediately reset the instance, as though it had stopped and immediately began to run again.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "reboot" - ] - } - }, - "required": [ - "type" - ] + "memory": { + "$ref": "#/components/schemas/ByteCount" + }, + "ncpus": { + "$ref": "#/components/schemas/InstanceCpuCount" } + }, + "required": [ + "hostname", + "memory", + "ncpus" ] }, - "InstanceUnregisterResponse": { - "description": "The response sent from a request to unregister an instance.", + "InstanceRuntimeState": { + "description": "The dynamic runtime properties of an instance: its current VMM ID (if any), migration information (if any), and the instance state to report if there is no active VMM.", "type": "object", "properties": { - "updated_runtime": { + "dst_propolis_id": { "nullable": true, - "description": "The current state of the instance after handling the request to unregister it. If the instance's state did not change, this field is `None`.", + "description": "If a migration is active, the ID of the target VMM.", + "allOf": [ + { + "$ref": "#/components/schemas/TypedUuidForPropolisKind" + } + ] + }, + "gen": { + "description": "Generation number for this state.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "migration_id": { + "nullable": true, + "description": "If a migration is active, the ID of that migration.", + "type": "string", + "format": "uuid" + }, + "propolis_id": { + "nullable": true, + "description": "The instance's currently active VMM ID.", "allOf": [ { - "$ref": "#/components/schemas/SledInstanceState" + "$ref": "#/components/schemas/TypedUuidForPropolisKind" } ] + }, + "time_updated": { + "description": "Timestamp for this information.", + "type": "string", + "format": "date-time" } - } + }, + "required": [ + "gen", + "time_updated" + ] }, "Inventory": { "description": "Identity and basic status information about this sled agent", @@ -4667,8 +4815,27 @@ "sled_id" ] }, - "SledInstanceState": { - "description": "A wrapper type containing a sled's total knowledge of the state of a specific VMM and the instance it incarnates.", + "SledRole": { + "description": "Describes the role of the sled within the rack.\n\nNote that this may change if the sled is physically moved within the rack.", + "oneOf": [ + { + "description": "The sled is a general compute sled.", + "type": "string", + "enum": [ + "gimlet" + ] + }, + { + "description": "The sled is attached to the network switch, and has additional responsibilities.", + "type": "string", + "enum": [ + "scrimlet" + ] + } + ] + }, + "SledVmmState": { + "description": "A wrapper type containing a sled's total knowledge of the state of a VMM.", "type": "object", "properties": { "migration_in": { @@ -4689,14 +4856,6 @@ } ] }, - "propolis_id": { - "description": "The ID of the VMM whose state is being reported.", - "allOf": [ - { - "$ref": "#/components/schemas/TypedUuidForPropolisKind" - } - ] - }, "vmm_state": { "description": "The most recent state of the sled's VMM process.", "allOf": [ @@ -4707,29 +4866,9 @@ } }, "required": [ - "propolis_id", "vmm_state" ] }, - "SledRole": { - "description": "Describes the role of the sled within the rack.\n\nNote that this may change if the sled is physically moved within the rack.", - "oneOf": [ - { - "description": "The sled is a general compute sled.", - "type": "string", - "enum": [ - "gimlet" - ] - }, - { - "description": "The sled is attached to the network switch, and has additional responsibilities.", - "type": "string", - "enum": [ - "scrimlet" - ] - } - ] - }, "Slot": { "description": "A stable index which is translated by Propolis into a PCI BDF, visible to the guest.\n\n
JSON schema\n\n```json { \"description\": \"A stable index which is translated by Propolis into a PCI BDF, visible to the guest.\", \"type\": \"integer\", \"format\": \"uint8\", \"minimum\": 0.0 } ```
", "type": "integer", @@ -4912,6 +5051,14 @@ "sync" ] }, + "TypedUuidForDatasetKind": { + "type": "string", + "format": "uuid" + }, + "TypedUuidForInstanceKind": { + "type": "string", + "format": "uuid" + }, "TypedUuidForPropolisKind": { "type": "string", "format": "uuid" @@ -4996,6 +5143,62 @@ "vni" ] }, + "VmmIssueDiskSnapshotRequestBody": { + "type": "object", + "properties": { + "snapshot_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "snapshot_id" + ] + }, + "VmmIssueDiskSnapshotRequestResponse": { + "type": "object", + "properties": { + "snapshot_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "snapshot_id" + ] + }, + "VmmPutStateBody": { + "description": "The body of a request to move a previously-ensured instance into a specific runtime state.", + "type": "object", + "properties": { + "state": { + "description": "The state into which the instance should be driven.", + "allOf": [ + { + "$ref": "#/components/schemas/VmmStateRequested" + } + ] + } + }, + "required": [ + "state" + ] + }, + "VmmPutStateResponse": { + "description": "The response sent from a request to move an instance into a specific runtime state.", + "type": "object", + "properties": { + "updated_runtime": { + "nullable": true, + "description": "The current runtime state of the instance after handling the request to change its state. If the instance's state did not change, this field is `None`.", + "allOf": [ + { + "$ref": "#/components/schemas/SledVmmState" + } + ] + } + } + }, "VmmRuntimeState": { "description": "The dynamic runtime properties of an individual VMM process.", "type": "object", @@ -5089,6 +5292,90 @@ } ] }, + "VmmStateRequested": { + "description": "Requestable running state of an Instance.\n\nA subset of [`omicron_common::api::external::InstanceState`].", + "oneOf": [ + { + "description": "Run this instance by migrating in from a previous running incarnation of the instance.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "migration_target" + ] + }, + "value": { + "$ref": "#/components/schemas/InstanceMigrationTargetParams" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Start the instance if it is not already running.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "running" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Stop the instance.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "stopped" + ] + } + }, + "required": [ + "type" + ] + }, + { + "description": "Immediately reset the instance, as though it had stopped and immediately began to run again.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "reboot" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, + "VmmUnregisterResponse": { + "description": "The response sent from a request to unregister an instance.", + "type": "object", + "properties": { + "updated_runtime": { + "nullable": true, + "description": "The current state of the instance after handling the request to unregister it. If the instance's state did not change, this field is `None`.", + "allOf": [ + { + "$ref": "#/components/schemas/SledVmmState" + } + ] + } + } + }, "Vni": { "description": "A Geneve Virtual Network Identifier", "type": "integer", @@ -5408,10 +5695,6 @@ "A", "B" ] - }, - "TypedUuidForInstanceKind": { - "type": "string", - "format": "uuid" } }, "responses": { diff --git a/oximeter/collector/src/bin/clickhouse-schema-updater.rs b/oximeter/collector/src/bin/clickhouse-schema-updater.rs index 20780c37e0..8e432e87c6 100644 --- a/oximeter/collector/src/bin/clickhouse-schema-updater.rs +++ b/oximeter/collector/src/bin/clickhouse-schema-updater.rs @@ -11,7 +11,7 @@ use anyhow::Context; use camino::Utf8PathBuf; use clap::Parser; use clap::Subcommand; -use omicron_common::address::CLICKHOUSE_PORT; +use omicron_common::address::CLICKHOUSE_HTTP_PORT; use oximeter_db::model::OXIMETER_VERSION; use oximeter_db::Client; use slog::Drain; @@ -24,7 +24,7 @@ use std::net::SocketAddrV6; const DEFAULT_HOST: SocketAddr = SocketAddr::V6(SocketAddrV6::new( Ipv6Addr::LOCALHOST, - CLICKHOUSE_PORT, + CLICKHOUSE_HTTP_PORT, 0, 0, )); diff --git a/oximeter/db/schema/replicated/12/timeseries-to-delete.txt b/oximeter/db/schema/replicated/12/timeseries-to-delete.txt new file mode 100644 index 0000000000..40b90e05ff --- /dev/null +++ b/oximeter/db/schema/replicated/12/timeseries-to-delete.txt @@ -0,0 +1 @@ +http_service:request_latency_histogram diff --git a/oximeter/db/schema/single-node/12/timeseries-to-delete.txt b/oximeter/db/schema/single-node/12/timeseries-to-delete.txt new file mode 100644 index 0000000000..40b90e05ff --- /dev/null +++ b/oximeter/db/schema/single-node/12/timeseries-to-delete.txt @@ -0,0 +1 @@ +http_service:request_latency_histogram diff --git a/oximeter/db/src/lib.rs b/oximeter/db/src/lib.rs index 5d56d802c9..2b3c2d6118 100644 --- a/oximeter/db/src/lib.rs +++ b/oximeter/db/src/lib.rs @@ -10,8 +10,6 @@ use crate::query::StringFieldSelector; use anyhow::Context as _; use chrono::DateTime; use chrono::Utc; -use dropshot::EmptyScanParams; -use dropshot::PaginationParams; pub use oximeter::schema::FieldSchema; pub use oximeter::schema::FieldSource; use oximeter::schema::TimeseriesKey; @@ -235,10 +233,6 @@ impl From for DbFieldSource { } } -/// Type used to paginate request to list timeseries schema. -pub type TimeseriesSchemaPaginationParams = - PaginationParams; - #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct TimeseriesScanParams { pub timeseries_name: TimeseriesName, diff --git a/oximeter/db/src/model.rs b/oximeter/db/src/model.rs index a3e9d109ff..d57819b0d0 100644 --- a/oximeter/db/src/model.rs +++ b/oximeter/db/src/model.rs @@ -45,7 +45,7 @@ use uuid::Uuid; /// - [`crate::Client::initialize_db_with_version`] /// - [`crate::Client::ensure_schema`] /// - The `clickhouse-schema-updater` binary in this crate -pub const OXIMETER_VERSION: u64 = 11; +pub const OXIMETER_VERSION: u64 = 12; // Wrapper type to represent a boolean in the database. // diff --git a/oximeter/instruments/src/http.rs b/oximeter/instruments/src/http.rs index 2eef327d02..efd053ad66 100644 --- a/oximeter/instruments/src/http.rs +++ b/oximeter/instruments/src/http.rs @@ -24,11 +24,11 @@ pub use http_service::RequestLatencyHistogram; impl RequestLatencyHistogram { /// Build a new `RequestLatencyHistogram` with a specified histogram. /// - /// Latencies are expressed in seconds. + /// Latencies are expressed in nanoseconds. pub fn new( operation_id: &str, status_code: StatusCode, - histogram: Histogram, + histogram: Histogram, ) -> Self { Self { operation_id: operation_id.to_string().into(), @@ -37,24 +37,26 @@ impl RequestLatencyHistogram { } } - /// Build a `RequestLatencyHistogram` with a histogram whose bins span the given decades. + /// Build a histogram whose bins span the given powers of ten. /// - /// `start_decade` and `end_decade` specify the lower and upper limits of the histogram's - /// range, as a power of 10. For example, passing `-3` and `2` results in a histogram with bins - /// spanning `[10 ** -3, 10 ** 2)`. There are 10 bins in each decade. See the - /// [`Histogram::span_decades`] method for more details. + /// `start_power` and `end_power` specify the lower and upper limits of + /// the histogram's range, as powers of 10. For example, passing 2 and 4 + /// results in a histogram with bins from `[10 ** 2, 10 ** 4)`. There are 10 + /// bins in each power of 10. /// - /// Latencies are expressed as seconds. - pub fn with_latency_decades( + /// See the [`Histogram::span_decades`] method for more details. + /// + /// Latencies are expressed in nanoseconds. + pub fn with_log_linear_bins( operation_id: &str, status_code: StatusCode, - start_decade: i16, - end_decade: i16, + start_power: u16, + end_power: u16, ) -> Result { Ok(Self::new( operation_id, status_code, - Histogram::span_decades(start_decade, end_decade)?, + Histogram::span_decades(start_power, end_power)?, )) } @@ -71,7 +73,7 @@ impl RequestLatencyHistogram { } /// The `LatencyTracker` is an [`oximeter::Producer`] that tracks the latencies of requests for an -/// HTTP service, in seconds. +/// HTTP service, in nanoseconds. /// /// Consumers should construct one `LatencyTracker` for each HTTP service they wish to instrument. /// As requests are received, the [`LatencyTracker::update`] method can be called with the @@ -94,14 +96,14 @@ pub struct LatencyTracker { /// The histogram used to track each request. /// /// We store it here to clone as we see new requests. - histogram: Histogram, + histogram: Histogram, } impl LatencyTracker { /// Build a new tracker for the given `service`, using `histogram` to track latencies. /// /// Note that the same histogram is used for each tracked timeseries. - pub fn new(service: HttpService, histogram: Histogram) -> Self { + pub fn new(service: HttpService, histogram: Histogram) -> Self { Self { service, latencies: Arc::new(Mutex::new(HashMap::new())), @@ -109,18 +111,19 @@ impl LatencyTracker { } } - /// Build a new tracker for the given `service`, with a histogram that spans the given decades - /// (powers of 10). See [`RequestLatencyHistogram::with_latency_decades`] for details on the + /// Build a new tracker with log-linear bins. + /// + /// This creates a tracker for the `service`, using 10 bins per power of 10, + /// from `[10 ** start_power, 10 ** end_power)`. + /// + /// [`RequestLatencyHistogram::with_log_linear_bins`] for details on the /// arguments. - pub fn with_latency_decades( + pub fn with_log_linear_bins( service: HttpService, - start_decade: i16, - end_decade: i16, + start_power: u16, + end_power: u16, ) -> Result { - Ok(Self::new( - service, - Histogram::span_decades(start_decade, end_decade)?, - )) + Ok(Self::new(service, Histogram::span_decades(start_power, end_power)?)) } /// Update (or create) a timeseries in response to a new request. @@ -142,7 +145,7 @@ impl LatencyTracker { self.histogram.clone(), ) }); - entry.datum.sample(latency.as_secs_f64()).map_err(MetricsError::from) + entry.datum.sample(latency.as_nanos() as _).map_err(MetricsError::from) } /// Instrument the given Dropshot endpoint handler function. @@ -218,16 +221,16 @@ mod tests { fn test_latency_tracker() { let service = HttpService { name: "my-service".into(), id: ID.parse().unwrap() }; - let hist = Histogram::new(&[0.0, 1.0]).unwrap(); + let hist = Histogram::new(&[100, 1000]).unwrap(); let tracker = LatencyTracker::new(service, hist); let status_code0 = StatusCode::OK; let status_code1 = StatusCode::NOT_FOUND; let operation_id = "some_operation_id"; tracker - .update(operation_id, status_code0, Duration::from_secs_f64(0.5)) + .update(operation_id, status_code0, Duration::from_nanos(200)) .unwrap(); tracker - .update(operation_id, status_code1, Duration::from_secs_f64(0.5)) + .update(operation_id, status_code1, Duration::from_nanos(200)) .unwrap(); let key0 = RequestLatencyHistogram::key_for(operation_id, status_code0); let key1 = RequestLatencyHistogram::key_for(operation_id, status_code1); diff --git a/oximeter/oximeter/schema/http-service.toml b/oximeter/oximeter/schema/http-service.toml index 5270f6942c..2e2e6fb359 100644 --- a/oximeter/oximeter/schema/http-service.toml +++ b/oximeter/oximeter/schema/http-service.toml @@ -11,8 +11,8 @@ versions = [ [[metrics]] name = "request_latency_histogram" description = "Duration for the server to handle a request" -units = "seconds" -datum_type = "histogram_f64" +units = "nanoseconds" +datum_type = "histogram_u64" versions = [ { added_in = 1, fields = [ "operation_id", "status_code" ] } ] diff --git a/oximeter/oximeter/schema/virtual-disk.toml b/oximeter/oximeter/schema/virtual-disk.toml new file mode 100644 index 0000000000..54cedae6e6 --- /dev/null +++ b/oximeter/oximeter/schema/virtual-disk.toml @@ -0,0 +1,127 @@ +format_version = 1 + +[target] +name = "virtual_disk" +description = "A virtual disk" +authz_scope = "project" +versions = [ + { version = 1, fields = [ "attached_instance_id", "block_size", "disk_id", "project_id", "silo_id", ] }, +] + +[fields.attached_instance_id] +type = "uuid" +description = "ID of the instance the disk is attached to" + +[fields.block_size] +type = "u32" +description = "Block size of the disk, in bytes" + +[fields.disk_id] +type = "uuid" +description = "ID of the disk" + +[fields.failure_reason] +type = "string" +description = "The reason an I/O operation failed" + +[fields.io_kind] +type = "string" +description = "The kind of I/O operation" + +[fields.project_id] +type = "uuid" +description = "ID of the project containing the disk" + +[fields.silo_id] +type = "uuid" +description = "ID for the silo containing the disk" + +[[metrics]] +name = "bytes_read" +description = "Number of bytes read from the disk" +units = "bytes" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "reads" +description = "Total number of read operations from the disk" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "failed_reads" +description = "Total number of failed read operations from the disk" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [ "failure_reason" ] } +] + +[[metrics]] +name = "bytes_written" +description = "Number of bytes written to the disk" +units = "bytes" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "writes" +description = "Total number of write operations to the disk" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "failed_writes" +description = "Total number of failed write operations to the disk" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [ "failure_reason" ] } +] + +[[metrics]] +name = "flushes" +description = "Total number of flush operations on the disk" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [] } +] + +[[metrics]] +name = "failed_flushes" +description = "Total number of failed flush operations on the disk" +units = "count" +datum_type = "cumulative_u64" +versions = [ + { added_in = 1, fields = [ "failure_reason" ] } +] + +[[metrics]] +name = "io_latency" +description = "Histogram of latency for I/O operations by kind" +units = "nanoseconds" +datum_type = "histogram_u64" +versions = [ + { added_in = 1, fields = [ "io_kind" ] } +] + +[[metrics]] +name = "io_size" +description = "Histogram of sizes for I/O operations by kind" +units = "bytes" +datum_type = "histogram_u64" +versions = [ + { added_in = 1, fields = [ "io_kind" ] } +] diff --git a/oximeter/types/src/histogram.rs b/oximeter/types/src/histogram.rs index 2a4feab382..2507f2f5c6 100644 --- a/oximeter/types/src/histogram.rs +++ b/oximeter/types/src/histogram.rs @@ -1191,6 +1191,47 @@ where } } +pub trait Bits: Integer { + const BITS: u32; + fn next_power(self) -> Option; +} + +macro_rules! impl_bits { + ($type_:ty) => { + impl Bits for $type_ { + const BITS: u32 = Self::BITS; + + fn next_power(self) -> Option { + self.checked_mul(2) + } + } + }; +} + +impl_bits!(u8); +impl_bits!(u16); +impl_bits!(u32); +impl_bits!(u64); + +impl Histogram +where + T: Bits + HistogramSupport, +{ + /// Create a histogram with logarithmically spaced bins at each power of 2. + /// + /// This is only available for unsigned integer support types. + pub fn power_of_two() -> Self { + let mut bins = Vec::with_capacity(T::BITS as _); + let mut x = T::one(); + bins.push(x); + while let Some(next) = x.next_power() { + bins.push(next); + x = next; + } + Self::new(&bins).expect("Bits is statically known") + } +} + // Helper to ensure all values are comparable, i.e., not NaN. fn ensure_finite(value: T) -> Result<(), HistogramError> where @@ -1797,4 +1838,19 @@ mod tests { "expected to overflow a u8, since support type is not wide enough", ); } + + #[test] + fn test_log_bins_u8() { + let (bins, _) = Histogram::::power_of_two().bins_and_counts(); + assert_eq!(bins, [0, 1, 2, 4, 8, 16, 32, 64, 128],); + } + + #[test] + fn test_log_bins_u64() { + let (bins, _) = Histogram::::power_of_two().bins_and_counts(); + assert_eq!(bins[0], 0); + for (i, bin) in bins.iter().skip(1).enumerate() { + assert_eq!(*bin, 1u64 << i); + } + } } diff --git a/schema/crdb/README.adoc b/schema/crdb/README.adoc index e017c01316..3567821ea6 100644 --- a/schema/crdb/README.adoc +++ b/schema/crdb/README.adoc @@ -126,19 +126,47 @@ same `NEW_VERSION`:**, then your `OLD_VERSION` has changed and so _your_ new version that came in from "main"). * Update the version in `dbinit.sql` to match the new `NEW_VERSION`. -=== General notes +=== Constraints on Schema Updates -CockroachDB's representation of the schema includes some opaque -internally-generated fields that are order dependent, like the names of -anonymous CHECK constraints. Our schema comparison tools intentionally ignore -these values. As a result, when performing schema changes, the order of new -tables and constraints should generally not be important. +==== Adding a new column without a default value [[add_column_constraint]] + +When adding a new non-nullable column to an existing table, that column must +contain a default to help back-fill existing rows in that table which may +exist. Without this default value, the schema upgrade might fail with +an error like `null value in column "..." violates not-null constraint`. +Unfortunately, it's possible that the schema upgrade might NOT fail with that +error, if no rows are present in the table when the schema is updated. This +results in an inconsistent state, where the schema upgrade might succeed on +some deployments but fail on others. + +If you'd like to add a column without a default value, we recommend +doing the following, if a `DEFAULT` value makes sense for a one-time update: + +1. Adding the column with a `DEFAULT` value. +2. Dropping the `DEFAULT` constraint. + +If a `DEFAULT` value does not make sense, then you need to implement a +multi-step process. + +. Add the column without a `NOT NULL` constraint. +. Migrate existing data to a non-null value. +. Once all data has been migrated to a non-null value, alter the table again to +add the `NOT NULL` constraint. -As convention, however, we recommend keeping the `db_metadata` file at the end -of `dbinit.sql`, so that the database does not contain a version until it is -fully populated. +For the time being, if you can write the data migration in SQL (e.g., using a +SQL `UPDATE`), then you can do this with a single new schema version where the +second step is an `UPDATE`. See schema version 54 (`blueprint-add-external-ip-id`) +for an example of this (though that one did not make the new column `NOT NULL` -- +you'd want to do that in another step). Update the `validate_data_migration()` +test in `nexus/tests/integration_tests/schema.rs` to add a test for this. -=== Scenario-specific gotchas +In the future when schema updates happen while the control plane is online, +this may not be a tenable path because the operation may take a very long time +on large tables. + +If you cannot write the data migration in SQL, you would need to figure out a +different way to backfill the data before you can apply the step that adds the +`NOT NULL` constraint. This is likely a substantial project ==== Renaming columns @@ -151,3 +179,66 @@ functions as a workaround.) An (imperfect) workaround is to use the `#[diesel(column_name = foo)]` attribute in Rust code to preserve the existing name of a column in the database while giving its corresponding struct field a different, more meaningful name. + +Note that this constraint does not apply to renaming tables: the statement +`ALTER TABLE IF EXISTS ... RENAME TO ...` is valid and idempotent. + +=== Fixing broken Schema Updates + +WARNING: This section is somewhat speculative - what "broken" means may differ +significantly from one schema update to the next. Take this as as a recommendation +based on experience, but not as a hard truth that will apply to all broken schema +updates. + +In cases where a schema update cannot complete successfully, additional steps +may be necessary to enable schema updates to proceed (for example, if a schema +update tried <>). In these situations, the goal should be +the following: + +. Fix the schema update such that deployments which have not applied it yet +do not fail. +.. It is important to update the *exact* "upN.sql" file which failed, rather than +re-numbering or otherwise changing the order of schema updates. Internally, Nexus +tracks which individual step of a schema update has been applied, to avoid applying +older schema upgrades which may no longer be relevant. +. Add a follow-up named schema update to ensure that deployments which have +*already* applied it arrive at the same state. This is only necessary if it is +possible for the schema update to apply successfully in any possible +deployment. This schema update should be added like any other "new" schema update, +appended to the list of all updates, rather than re-ordering history. This +schema update will run on systems that deployed both versions of the earlier +schema update. +. Determine whether any of the schema versions after the broken one need to +change because they depended on the specific behavior that you changed to _fix_ +that version. + +We can use the following terminology here: + +* `S(bad)`: The particular `upN.sql` schema update which is "broken". +* `S(fixed)`: That same `upN.sql` file after being updated to a non-broken version. +* `S(converge)`: Some later schema update that converges the deployment to a known-good +state. + +**This process is risky**. By changing the contents of the old schema update `S(bad)` +to `S(fixed)`, we create two divergent histories on our deployments: one where `S(bad)` +may have been applied, and one where only `S(fixed)` was applied. + +Although the goal of `S(converge)` is to make sure that these deployments end +up looking the same, there are no guarantees that other schema updates between +`S(bad)` and `S(converge)` will be identical between these two variant update +timelines. When fixing broken schema updates, do so with caution, and consider +all schema updates between `S(bad)` and `S(converge)` - these updates must be +able to complete successfully regardless of which one of `S(bad)` or `S(fixed)` +was applied. + +=== General notes + +CockroachDB's representation of the schema includes some opaque +internally-generated fields that are order dependent, like the names of +anonymous CHECK constraints. Our schema comparison tools intentionally ignore +these values. As a result, when performing schema changes, the order of new +tables and constraints should generally not be important. + +As convention, however, we recommend keeping the `db_metadata` row insertion at +the end of `dbinit.sql`, so that the database does not contain a version until +it is fully populated. diff --git a/schema/crdb/dataset-kinds-zone-and-debug/up01.sql b/schema/crdb/dataset-kinds-zone-and-debug/up01.sql new file mode 100644 index 0000000000..1cfe718d00 --- /dev/null +++ b/schema/crdb/dataset-kinds-zone-and-debug/up01.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.dataset_kind ADD VALUE IF NOT EXISTS 'zone_root' AFTER 'internal_dns'; diff --git a/schema/crdb/dataset-kinds-zone-and-debug/up02.sql b/schema/crdb/dataset-kinds-zone-and-debug/up02.sql new file mode 100644 index 0000000000..93178e3685 --- /dev/null +++ b/schema/crdb/dataset-kinds-zone-and-debug/up02.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.dataset_kind ADD VALUE IF NOT EXISTS 'zone' AFTER 'zone_root'; diff --git a/schema/crdb/dataset-kinds-zone-and-debug/up03.sql b/schema/crdb/dataset-kinds-zone-and-debug/up03.sql new file mode 100644 index 0000000000..58d215d177 --- /dev/null +++ b/schema/crdb/dataset-kinds-zone-and-debug/up03.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.dataset_kind ADD VALUE IF NOT EXISTS 'debug' AFTER 'zone'; diff --git a/schema/crdb/dataset-kinds-zone-and-debug/up04.sql b/schema/crdb/dataset-kinds-zone-and-debug/up04.sql new file mode 100644 index 0000000000..b92bce1b6c --- /dev/null +++ b/schema/crdb/dataset-kinds-zone-and-debug/up04.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.dataset ADD COLUMN IF NOT EXISTS zone_name TEXT; diff --git a/schema/crdb/dataset-kinds-zone-and-debug/up05.sql b/schema/crdb/dataset-kinds-zone-and-debug/up05.sql new file mode 100644 index 0000000000..3f33b79c72 --- /dev/null +++ b/schema/crdb/dataset-kinds-zone-and-debug/up05.sql @@ -0,0 +1,4 @@ +ALTER TABLE omicron.public.dataset ADD CONSTRAINT IF NOT EXISTS zone_name_for_zone_kind CHECK ( + (kind != 'zone') OR + (kind = 'zone' AND zone_name IS NOT NULL) +) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index d531672832..e851d2ed6b 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -509,7 +509,10 @@ CREATE TYPE IF NOT EXISTS omicron.public.dataset_kind AS ENUM ( 'clickhouse_keeper', 'clickhouse_server', 'external_dns', - 'internal_dns' + 'internal_dns', + 'zone_root', + 'zone', + 'debug' ); /* @@ -535,6 +538,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.dataset ( /* An upper bound on the amount of space that might be in-use */ size_used INT, + /* Only valid if kind = zone -- the name of this zone */ + zone_name TEXT, + /* Crucible must make use of 'size_used'; other datasets manage their own storage */ CONSTRAINT size_used_column_set_for_crucible CHECK ( (kind != 'crucible') OR @@ -544,6 +550,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.dataset ( CONSTRAINT ip_and_port_set_for_crucible CHECK ( (kind != 'crucible') OR (kind = 'crucible' AND ip IS NOT NULL and port IS NOT NULL) + ), + + CONSTRAINT zone_name_for_zone_kind CHECK ( + (kind != 'zone') OR + (kind = 'zone' AND zone_name IS NOT NULL) ) ); @@ -4214,7 +4225,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '92.0.0', NULL) + (TRUE, NOW(), NOW(), '93.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/omicron-datasets.json b/schema/omicron-datasets.json new file mode 100644 index 0000000000..07fc2cfb13 --- /dev/null +++ b/schema/omicron-datasets.json @@ -0,0 +1,226 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DatasetsConfig", + "type": "object", + "required": [ + "datasets", + "generation" + ], + "properties": { + "datasets": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/DatasetConfig" + } + }, + "generation": { + "description": "generation number of this configuration\n\nThis generation number is owned by the control plane (i.e., RSS or Nexus, depending on whether RSS-to-Nexus handoff has happened). It should not be bumped within Sled Agent.\n\nSled Agent rejects attempts to set the configuration to a generation older than the one it's currently running.\n\nNote that \"Generation::new()\", AKA, the first generation number, is reserved for \"no datasets\". This is the default configuration for a sled before any requests have been made.", + "allOf": [ + { + "$ref": "#/definitions/Generation" + } + ] + } + }, + "definitions": { + "CompressionAlgorithm": { + "oneOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "on" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "off" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "gzip" + ] + } + } + }, + { + "type": "object", + "required": [ + "level", + "type" + ], + "properties": { + "level": { + "$ref": "#/definitions/GzipLevel" + }, + "type": { + "type": "string", + "enum": [ + "gzip_n" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "lz4" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "lzjb" + ] + } + } + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "zle" + ] + } + } + } + ] + }, + "DatasetConfig": { + "description": "Configuration information necessary to request a single dataset", + "type": "object", + "required": [ + "compression", + "id", + "name" + ], + "properties": { + "compression": { + "description": "The compression mode to be used by the dataset", + "allOf": [ + { + "$ref": "#/definitions/CompressionAlgorithm" + } + ] + }, + "id": { + "description": "The UUID of the dataset being requested", + "allOf": [ + { + "$ref": "#/definitions/TypedUuidForDatasetKind" + } + ] + }, + "name": { + "description": "The dataset's name", + "allOf": [ + { + "$ref": "#/definitions/DatasetName" + } + ] + }, + "quota": { + "description": "The upper bound on the amount of storage used by this dataset", + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 0.0 + }, + "reservation": { + "description": "The lower bound on the amount of storage usable by this dataset", + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 0.0 + } + } + }, + "DatasetKind": { + "description": "The kind of dataset. See the `DatasetKind` enum in omicron-common for possible values.", + "type": "string" + }, + "DatasetName": { + "type": "object", + "required": [ + "kind", + "pool_name" + ], + "properties": { + "kind": { + "$ref": "#/definitions/DatasetKind" + }, + "pool_name": { + "$ref": "#/definitions/ZpoolName" + } + } + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "GzipLevel": { + "type": "integer", + "format": "uint8", + "minimum": 0.0 + }, + "TypedUuidForDatasetKind": { + "type": "string", + "format": "uuid" + }, + "ZpoolName": { + "title": "The name of a Zpool", + "description": "Zpool names are of the format ox{i,p}_. They are either Internal or External, and should be unique", + "type": "string", + "pattern": "^ox[ip]_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + } + } +} \ No newline at end of file diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index c44b24d712..d9e49a5c56 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -15,15 +15,18 @@ use nexus_sled_agent_shared::inventory::{ }; use omicron_common::{ api::internal::{ - nexus::{DiskRuntimeState, SledInstanceState, UpdateArtifactId}, + nexus::{DiskRuntimeState, SledVmmState, UpdateArtifactId}, shared::{ ResolvedVpcRouteSet, ResolvedVpcRouteState, SledIdentifiers, SwitchPorts, VirtualNetworkInterfaceHost, }, }, - disk::{DiskVariant, DisksManagementResult, OmicronPhysicalDisksConfig}, + disk::{ + DatasetsConfig, DatasetsManagementResult, DiskVariant, + DisksManagementResult, OmicronPhysicalDisksConfig, + }, }; -use omicron_uuid_kinds::{InstanceUuid, ZpoolUuid}; +use omicron_uuid_kinds::{PropolisUuid, ZpoolUuid}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_agent_types::{ @@ -36,8 +39,8 @@ use sled_agent_types::{ early_networking::EarlyNetworkConfig, firewall_rules::VpcFirewallRulesEnsureBody, instance::{ - InstanceEnsureBody, InstanceExternalIpBody, InstancePutStateBody, - InstancePutStateResponse, InstanceUnregisterResponse, + InstanceEnsureBody, InstanceExternalIpBody, VmmPutStateBody, + VmmPutStateResponse, VmmUnregisterResponse, }, sled::AddSledRequest, time_sync::TimeSync, @@ -168,6 +171,25 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result; + /// Configures datasets to be used on this sled + #[endpoint { + method = PUT, + path = "/datasets", + }] + async fn datasets_put( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError>; + + /// Lists the datasets that this sled is configured to use + #[endpoint { + method = GET, + path = "/datasets", + }] + async fn datasets_get( + rqctx: RequestContext, + ) -> Result, HttpError>; + #[endpoint { method = GET, path = "/omicron-physical-disks", @@ -212,59 +234,59 @@ pub trait SledAgentApi { #[endpoint { method = PUT, - path = "/instances/{instance_id}", + path = "/vmms/{propolis_id}", }] - async fn instance_register( + async fn vmm_register( rqctx: RequestContext, - path_params: Path, + path_params: Path, body: TypedBody, - ) -> Result, HttpError>; + ) -> Result, HttpError>; #[endpoint { method = DELETE, - path = "/instances/{instance_id}", + path = "/vmms/{propolis_id}", }] - async fn instance_unregister( + async fn vmm_unregister( rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError>; + path_params: Path, + ) -> Result, HttpError>; #[endpoint { method = PUT, - path = "/instances/{instance_id}/state", + path = "/vmms/{propolis_id}/state", }] - async fn instance_put_state( + async fn vmm_put_state( rqctx: RequestContext, - path_params: Path, - body: TypedBody, - ) -> Result, HttpError>; + path_params: Path, + body: TypedBody, + ) -> Result, HttpError>; #[endpoint { method = GET, - path = "/instances/{instance_id}/state", + path = "/vmms/{propolis_id}/state", }] - async fn instance_get_state( + async fn vmm_get_state( rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError>; + path_params: Path, + ) -> Result, HttpError>; #[endpoint { method = PUT, - path = "/instances/{instance_id}/external-ip", + path = "/vmms/{propolis_id}/external-ip", }] - async fn instance_put_external_ip( + async fn vmm_put_external_ip( rqctx: RequestContext, - path_params: Path, + path_params: Path, body: TypedBody, ) -> Result; #[endpoint { method = DELETE, - path = "/instances/{instance_id}/external-ip", + path = "/vmms/{propolis_id}/external-ip", }] - async fn instance_delete_external_ip( + async fn vmm_delete_external_ip( rqctx: RequestContext, - path_params: Path, + path_params: Path, body: TypedBody, ) -> Result; @@ -290,16 +312,13 @@ pub trait SledAgentApi { /// Take a snapshot of a disk that is attached to an instance #[endpoint { method = POST, - path = "/instances/{instance_id}/disks/{disk_id}/snapshot", + path = "/vmms/{propolis_id}/disks/{disk_id}/snapshot", }] - async fn instance_issue_disk_snapshot_request( + async fn vmm_issue_disk_snapshot_request( rqctx: RequestContext, - path_params: Path, - body: TypedBody, - ) -> Result< - HttpResponseOk, - HttpError, - >; + path_params: Path, + body: TypedBody, + ) -> Result, HttpError>; #[endpoint { method = PUT, @@ -516,8 +535,8 @@ impl From for DiskType { /// Path parameters for Instance requests (sled agent API) #[derive(Deserialize, JsonSchema)] -pub struct InstancePathParam { - pub instance_id: InstanceUuid, +pub struct VmmPathParam { + pub propolis_id: PropolisUuid, } /// Path parameters for Disk requests (sled agent API) @@ -527,18 +546,18 @@ pub struct DiskPathParam { } #[derive(Deserialize, JsonSchema)] -pub struct InstanceIssueDiskSnapshotRequestPathParam { - pub instance_id: Uuid, +pub struct VmmIssueDiskSnapshotRequestPathParam { + pub propolis_id: PropolisUuid, pub disk_id: Uuid, } #[derive(Deserialize, JsonSchema)] -pub struct InstanceIssueDiskSnapshotRequestBody { +pub struct VmmIssueDiskSnapshotRequestBody { pub snapshot_id: Uuid, } #[derive(Serialize, JsonSchema)] -pub struct InstanceIssueDiskSnapshotRequestResponse { +pub struct VmmIssueDiskSnapshotRequestResponse { pub snapshot_id: Uuid, } diff --git a/sled-agent/src/backing_fs.rs b/sled-agent/src/backing_fs.rs index 2e9ea4c8d9..a0f7826db3 100644 --- a/sled-agent/src/backing_fs.rs +++ b/sled-agent/src/backing_fs.rs @@ -25,6 +25,7 @@ use camino::Utf8PathBuf; use illumos_utils::zfs::{ EnsureFilesystemError, GetValueError, Mountpoint, SizeDetails, Zfs, }; +use omicron_common::disk::CompressionAlgorithm; use std::io; #[derive(Debug, thiserror::Error)] @@ -50,7 +51,7 @@ struct BackingFs<'a> { // Optional quota, in _bytes_ quota: Option, // Optional compression mode - compression: Option<&'static str>, + compression: CompressionAlgorithm, // Linked service service: Option<&'static str>, // Subdirectories to ensure @@ -63,7 +64,7 @@ impl<'a> BackingFs<'a> { name, mountpoint: "legacy", quota: None, - compression: None, + compression: CompressionAlgorithm::Off, service: None, subdirs: None, } @@ -79,8 +80,8 @@ impl<'a> BackingFs<'a> { self } - const fn compression(mut self, compression: &'static str) -> Self { - self.compression = Some(compression); + const fn compression(mut self, compression: CompressionAlgorithm) -> Self { + self.compression = compression; self } @@ -101,7 +102,7 @@ const BACKING_FMD_SUBDIRS: [&'static str; 3] = ["rsrc", "ckpt", "xprt"]; const BACKING_FMD_SERVICE: &'static str = "svc:/system/fmd:default"; const BACKING_FMD_QUOTA: usize = 500 * (1 << 20); // 500 MiB -const BACKING_COMPRESSION: &'static str = "on"; +const BACKING_COMPRESSION: CompressionAlgorithm = CompressionAlgorithm::On; const BACKINGFS_COUNT: usize = 1; static BACKINGFS: [BackingFs; BACKINGFS_COUNT] = @@ -137,6 +138,7 @@ pub(crate) fn ensure_backing_fs( let size_details = Some(SizeDetails { quota: bfs.quota, + reservation: None, compression: bfs.compression, }); diff --git a/sled-agent/src/common/instance.rs b/sled-agent/src/common/instance.rs index adbeb9158f..f95bf0cb64 100644 --- a/sled-agent/src/common/instance.rs +++ b/sled-agent/src/common/instance.rs @@ -7,10 +7,9 @@ use chrono::{DateTime, Utc}; use omicron_common::api::external::Generation; use omicron_common::api::internal::nexus::{ - MigrationRuntimeState, MigrationState, SledInstanceState, VmmRuntimeState, + MigrationRuntimeState, MigrationState, SledVmmState, VmmRuntimeState, VmmState, }; -use omicron_uuid_kinds::PropolisUuid; use propolis_client::types::{ InstanceMigrationStatus, InstanceState as PropolisApiState, InstanceStateMonitorResponse, MigrationState as PropolisMigrationState, @@ -21,7 +20,6 @@ use uuid::Uuid; #[derive(Clone, Debug)] pub struct InstanceStates { vmm: VmmRuntimeState, - propolis_id: PropolisUuid, migration_in: Option, migration_out: Option, } @@ -173,11 +171,7 @@ pub enum Action { } impl InstanceStates { - pub fn new( - vmm: VmmRuntimeState, - propolis_id: PropolisUuid, - migration_id: Option, - ) -> Self { + pub fn new(vmm: VmmRuntimeState, migration_id: Option) -> Self { // If this instance is created with a migration ID, we are the intended // target of a migration in. Set that up now. let migration_in = @@ -187,17 +181,13 @@ impl InstanceStates { gen: Generation::new(), time_updated: Utc::now(), }); - InstanceStates { vmm, propolis_id, migration_in, migration_out: None } + InstanceStates { vmm, migration_in, migration_out: None } } pub fn vmm(&self) -> &VmmRuntimeState { &self.vmm } - pub fn propolis_id(&self) -> PropolisUuid { - self.propolis_id - } - pub fn migration_in(&self) -> Option<&MigrationRuntimeState> { self.migration_in.as_ref() } @@ -209,10 +199,9 @@ impl InstanceStates { /// Creates a `SledInstanceState` structure containing the entirety of this /// structure's runtime state. This requires cloning; for simple read access /// use the `instance` or `vmm` accessors instead. - pub fn sled_instance_state(&self) -> SledInstanceState { - SledInstanceState { + pub fn sled_instance_state(&self) -> SledVmmState { + SledVmmState { vmm_state: self.vmm.clone(), - propolis_id: self.propolis_id, migration_in: self.migration_in.clone(), migration_out: self.migration_out.clone(), } @@ -377,7 +366,6 @@ mod test { use uuid::Uuid; fn make_instance() -> InstanceStates { - let propolis_id = PropolisUuid::new_v4(); let now = Utc::now(); let vmm = VmmRuntimeState { @@ -386,7 +374,7 @@ mod test { time_updated: now, }; - InstanceStates::new(vmm, propolis_id, None) + InstanceStates::new(vmm, None) } fn make_migration_source_instance() -> InstanceStates { @@ -406,7 +394,6 @@ mod test { } fn make_migration_target_instance() -> InstanceStates { - let propolis_id = PropolisUuid::new_v4(); let now = Utc::now(); let vmm = VmmRuntimeState { @@ -415,7 +402,7 @@ mod test { time_updated: now, }; - InstanceStates::new(vmm, propolis_id, Some(Uuid::new_v4())) + InstanceStates::new(vmm, Some(Uuid::new_v4())) } fn make_observed_state( diff --git a/sled-agent/src/fakes/nexus.rs b/sled-agent/src/fakes/nexus.rs index 246ef07b60..bd4680563e 100644 --- a/sled-agent/src/fakes/nexus.rs +++ b/sled-agent/src/fakes/nexus.rs @@ -15,12 +15,11 @@ use hyper::Body; use internal_dns::ServiceName; use nexus_client::types::SledAgentInfo; use omicron_common::api::external::Error; -use omicron_common::api::internal::nexus::{ - SledInstanceState, UpdateArtifactId, -}; -use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_common::api::internal::nexus::{SledVmmState, UpdateArtifactId}; +use omicron_uuid_kinds::{OmicronZoneUuid, PropolisUuid}; use schemars::JsonSchema; use serde::Deserialize; +use sled_agent_api::VmmPathParam; use uuid::Uuid; /// Implements a fake Nexus. @@ -50,8 +49,8 @@ pub trait FakeNexusServer: Send + Sync { fn cpapi_instances_put( &self, - _instance_id: Uuid, - _new_runtime_state: SledInstanceState, + _propolis_id: PropolisUuid, + _new_runtime_state: SledVmmState, ) -> Result<(), Error> { Err(Error::internal_error("Not implemented")) } @@ -118,22 +117,18 @@ async fn sled_agent_put( Ok(HttpResponseUpdatedNoContent()) } -#[derive(Deserialize, JsonSchema)] -struct InstancePathParam { - instance_id: Uuid, -} #[endpoint { method = PUT, - path = "/instances/{instance_id}", + path = "/vmms/{propolis_id}", }] async fn cpapi_instances_put( request_context: RequestContext, - path_params: Path, - new_runtime_state: TypedBody, + path_params: Path, + new_runtime_state: TypedBody, ) -> Result { let context = request_context.context(); context.cpapi_instances_put( - path_params.into_inner().instance_id, + path_params.into_inner().propolis_id, new_runtime_state.into_inner(), )?; Ok(HttpResponseUpdatedNoContent()) diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 2bf8067d1c..1d61d97675 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -21,16 +21,16 @@ use nexus_sled_agent_shared::inventory::{ }; use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::{ - DiskRuntimeState, SledInstanceState, UpdateArtifactId, + DiskRuntimeState, SledVmmState, UpdateArtifactId, }; use omicron_common::api::internal::shared::{ ResolvedVpcRouteSet, ResolvedVpcRouteState, SledIdentifiers, SwitchPorts, VirtualNetworkInterfaceHost, }; use omicron_common::disk::{ - DiskVariant, DisksManagementResult, M2Slot, OmicronPhysicalDisksConfig, + DatasetsConfig, DatasetsManagementResult, DiskVariant, + DisksManagementResult, M2Slot, OmicronPhysicalDisksConfig, }; -use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; use sled_agent_api::*; use sled_agent_types::boot_disk::{ BootDiskOsWriteStatus, BootDiskPathParams, BootDiskUpdatePathParams, @@ -41,8 +41,8 @@ use sled_agent_types::disk::DiskEnsureBody; use sled_agent_types::early_networking::EarlyNetworkConfig; use sled_agent_types::firewall_rules::VpcFirewallRulesEnsureBody; use sled_agent_types::instance::{ - InstanceEnsureBody, InstanceExternalIpBody, InstancePutStateBody, - InstancePutStateResponse, InstanceUnregisterResponse, + InstanceEnsureBody, InstanceExternalIpBody, VmmPutStateBody, + VmmPutStateResponse, VmmUnregisterResponse, }; use sled_agent_types::sled::AddSledRequest; use sled_agent_types::time_sync::TimeSync; @@ -220,6 +220,23 @@ impl SledAgentApi for SledAgentImpl { .map_err(HttpError::from) } + async fn datasets_put( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError> { + let sa = rqctx.context(); + let body_args = body.into_inner(); + let result = sa.datasets_ensure(body_args).await?; + Ok(HttpResponseOk(result)) + } + + async fn datasets_get( + rqctx: RequestContext, + ) -> Result, HttpError> { + let sa = rqctx.context(); + Ok(HttpResponseOk(sa.datasets_config_list().await?)) + } + async fn zone_bundle_cleanup( rqctx: RequestContext, ) -> Result>, HttpError> @@ -294,18 +311,18 @@ impl SledAgentApi for SledAgentImpl { Ok(HttpResponseUpdatedNoContent()) } - async fn instance_register( + async fn vmm_register( rqctx: RequestContext, - path_params: Path, + path_params: Path, body: TypedBody, - ) -> Result, HttpError> { + ) -> Result, HttpError> { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; + let propolis_id = path_params.into_inner().propolis_id; let body_args = body.into_inner(); Ok(HttpResponseOk( sa.instance_ensure_registered( - instance_id, - body_args.propolis_id, + body_args.instance_id, + propolis_id, body_args.hardware, body_args.instance_runtime, body_args.vmm_runtime, @@ -316,58 +333,56 @@ impl SledAgentApi for SledAgentImpl { )) } - async fn instance_unregister( + async fn vmm_unregister( rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError> { + path_params: Path, + ) -> Result, HttpError> { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; - Ok(HttpResponseOk(sa.instance_ensure_unregistered(instance_id).await?)) + let id = path_params.into_inner().propolis_id; + Ok(HttpResponseOk(sa.instance_ensure_unregistered(id).await?)) } - async fn instance_put_state( + async fn vmm_put_state( rqctx: RequestContext, - path_params: Path, - body: TypedBody, - ) -> Result, HttpError> { + path_params: Path, + body: TypedBody, + ) -> Result, HttpError> { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; + let id = path_params.into_inner().propolis_id; let body_args = body.into_inner(); - Ok(HttpResponseOk( - sa.instance_ensure_state(instance_id, body_args.state).await?, - )) + Ok(HttpResponseOk(sa.instance_ensure_state(id, body_args.state).await?)) } - async fn instance_get_state( + async fn vmm_get_state( rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError> { + path_params: Path, + ) -> Result, HttpError> { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; - Ok(HttpResponseOk(sa.instance_get_state(instance_id).await?)) + let id = path_params.into_inner().propolis_id; + Ok(HttpResponseOk(sa.instance_get_state(id).await?)) } - async fn instance_put_external_ip( + async fn vmm_put_external_ip( rqctx: RequestContext, - path_params: Path, + path_params: Path, body: TypedBody, ) -> Result { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; + let id = path_params.into_inner().propolis_id; let body_args = body.into_inner(); - sa.instance_put_external_ip(instance_id, &body_args).await?; + sa.instance_put_external_ip(id, &body_args).await?; Ok(HttpResponseUpdatedNoContent()) } - async fn instance_delete_external_ip( + async fn vmm_delete_external_ip( rqctx: RequestContext, - path_params: Path, + path_params: Path, body: TypedBody, ) -> Result { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; + let id = path_params.into_inner().propolis_id; let body_args = body.into_inner(); - sa.instance_delete_external_ip(instance_id, &body_args).await?; + sa.instance_delete_external_ip(id, &body_args).await?; Ok(HttpResponseUpdatedNoContent()) } @@ -399,26 +414,24 @@ impl SledAgentApi for SledAgentImpl { Ok(HttpResponseUpdatedNoContent()) } - async fn instance_issue_disk_snapshot_request( + async fn vmm_issue_disk_snapshot_request( rqctx: RequestContext, - path_params: Path, - body: TypedBody, - ) -> Result< - HttpResponseOk, - HttpError, - > { + path_params: Path, + body: TypedBody, + ) -> Result, HttpError> + { let sa = rqctx.context(); let path_params = path_params.into_inner(); let body = body.into_inner(); - sa.instance_issue_disk_snapshot_request( - InstanceUuid::from_untyped_uuid(path_params.instance_id), + sa.vmm_issue_disk_snapshot_request( + path_params.propolis_id, path_params.disk_id, body.snapshot_id, ) .await?; - Ok(HttpResponseOk(InstanceIssueDiskSnapshotRequestResponse { + Ok(HttpResponseOk(VmmIssueDiskSnapshotRequestResponse { snapshot_id: body.snapshot_id, })) } diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 0bcbc97fd2..33b2d0cf67 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -25,14 +25,13 @@ use illumos_utils::opte::{DhcpCfg, PortCreateParams, PortManager}; use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory}; use illumos_utils::svc::wait_for_service; use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; -use omicron_common::api::internal::nexus::{ - SledInstanceState, VmmRuntimeState, -}; +use omicron_common::api::internal::nexus::{SledVmmState, VmmRuntimeState}; use omicron_common::api::internal::shared::{ NetworkInterface, ResolvedVpcFirewallRule, SledIdentifiers, SourceNatConfig, }; use omicron_common::backoff; use omicron_common::zpool_name::ZpoolName; +use omicron_common::NoDebug; use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid}; use propolis_client::Client as PropolisClient; use rand::prelude::IteratorRandom; @@ -104,11 +103,11 @@ pub enum Error { #[error("Error resolving DNS name: {0}")] ResolveError(#[from] internal_dns::resolver::ResolveError), - #[error("Instance {0} not running!")] - InstanceNotRunning(InstanceUuid), + #[error("Propolis job with ID {0} is registered but not running")] + VmNotRunning(PropolisUuid), - #[error("Instance already registered with Propolis ID {0}")] - InstanceAlreadyRegistered(PropolisUuid), + #[error("Propolis job with ID {0} already registered")] + PropolisAlreadyRegistered(PropolisUuid), #[error("No U.2 devices found")] U2NotFound, @@ -217,15 +216,15 @@ enum InstanceRequest { tx: oneshot::Sender>, }, CurrentState { - tx: oneshot::Sender, + tx: oneshot::Sender, }, PutState { - state: InstanceStateRequested, - tx: oneshot::Sender>, + state: VmmStateRequested, + tx: oneshot::Sender>, }, Terminate { mark_failed: bool, - tx: oneshot::Sender>, + tx: oneshot::Sender>, }, IssueSnapshotRequest { disk_id: Uuid, @@ -337,7 +336,7 @@ struct InstanceRunner { // Disk related properties requested_disks: Vec, - cloud_init_bytes: Option, + cloud_init_bytes: Option>, // Internal State management state: InstanceStates, @@ -414,12 +413,12 @@ impl InstanceRunner { }, Some(PutState{ state, tx }) => { tx.send(self.put_state(state).await - .map(|r| InstancePutStateResponse { updated_runtime: Some(r) }) + .map(|r| VmmPutStateResponse { updated_runtime: Some(r) }) .map_err(|e| e.into())) .map_err(|_| Error::FailedSendClientClosed) }, Some(Terminate { mark_failed, tx }) => { - tx.send(Ok(InstanceUnregisterResponse { + tx.send(Ok(VmmUnregisterResponse { updated_runtime: Some(self.terminate(mark_failed).await) })) .map_err(|_| Error::FailedSendClientClosed) @@ -499,15 +498,10 @@ impl InstanceRunner { } /// Yields this instance's ID. - fn id(&self) -> InstanceUuid { + fn instance_id(&self) -> InstanceUuid { InstanceUuid::from_untyped_uuid(self.properties.id) } - /// Yields this instance's Propolis's ID. - fn propolis_id(&self) -> &PropolisUuid { - &self.propolis_id - } - async fn publish_state_to_nexus(&self) { // Retry until Nexus acknowledges that it has applied this state update. // Note that Nexus may receive this call but then fail while reacting @@ -518,15 +512,13 @@ impl InstanceRunner { || async { let state = self.state.sled_instance_state(); info!(self.log, "Publishing instance state update to Nexus"; - "instance_id" => %self.id(), + "instance_id" => %self.instance_id(), + "propolis_id" => %self.propolis_id, "state" => ?state, ); self.nexus_client - .cpapi_instances_put( - &self.id().into_untyped_uuid(), - &state.into(), - ) + .cpapi_instances_put(&self.propolis_id, &state.into()) .await .map_err(|err| -> backoff::BackoffError { match &err { @@ -576,7 +568,8 @@ impl InstanceRunner { warn!(self.log, "Failed to publish instance state to Nexus: {}", err.to_string(); - "instance_id" => %self.id(), + "instance_id" => %self.instance_id(), + "propolis_id" => %self.propolis_id, "retry_after" => ?delay); }, ) @@ -586,7 +579,8 @@ impl InstanceRunner { error!( self.log, "Failed to publish state to Nexus, will not retry: {:?}", e; - "instance_id" => %self.id() + "instance_id" => %self.instance_id(), + "propolis_id" => %self.propolis_id, ); } } @@ -622,7 +616,7 @@ impl InstanceRunner { info!( self.log, "updated state after observing Propolis state change"; - "propolis_id" => %self.state.propolis_id(), + "propolis_id" => %self.propolis_id, "new_vmm_state" => ?self.state.vmm() ); @@ -634,7 +628,8 @@ impl InstanceRunner { match action { Some(InstanceAction::Destroy) => { info!(self.log, "terminating VMM that has exited"; - "instance_id" => %self.id()); + "instance_id" => %self.instance_id(), + "propolis_id" => %self.propolis_id); let mark_failed = false; self.terminate(mark_failed).await; Reaction::Terminate @@ -724,10 +719,10 @@ impl InstanceRunner { .map(Into::into) .collect(), migrate, - cloud_init_bytes: self.cloud_init_bytes.clone(), + cloud_init_bytes: self.cloud_init_bytes.clone().map(|x| x.0), }; - info!(self.log, "Sending ensure request to propolis: {:?}", request); + debug!(self.log, "Sending ensure request to propolis: {:?}", request); let result = client.instance_ensure().body(request).send().await; info!(self.log, "result of instance_ensure call is {:?}", result); result?; @@ -780,7 +775,7 @@ impl InstanceRunner { /// This routine is safe to call even if the instance's zone was never /// started. It is also safe to call multiple times on a single instance. async fn terminate_inner(&mut self) { - let zname = propolis_zone_name(self.propolis_id()); + let zname = propolis_zone_name(&self.propolis_id); // First fetch the running state. // @@ -948,8 +943,10 @@ impl InstanceRunner { } } -/// A reference to a single instance running a running Propolis server. +/// Describes a single Propolis server that incarnates a specific instance. pub struct Instance { + id: InstanceUuid, + tx: mpsc::Sender, #[allow(dead_code)] @@ -1091,7 +1088,7 @@ impl Instance { dhcp_config, requested_disks: hardware.disks, cloud_init_bytes: hardware.cloud_init_bytes, - state: InstanceStates::new(vmm_runtime, propolis_id, migration_id), + state: InstanceStates::new(vmm_runtime, migration_id), running_state: None, nexus_client, storage, @@ -1104,7 +1101,11 @@ impl Instance { let runner_handle = tokio::task::spawn(async move { runner.run().await }); - Ok(Instance { tx, runner_handle }) + Ok(Instance { id, tx, runner_handle }) + } + + pub fn id(&self) -> InstanceUuid { + self.id } /// Create bundle from an instance zone. @@ -1130,7 +1131,7 @@ impl Instance { Ok(rx.await?) } - pub async fn current_state(&self) -> Result { + pub async fn current_state(&self) -> Result { let (tx, rx) = oneshot::channel(); self.tx .send(InstanceRequest::CurrentState { tx }) @@ -1152,8 +1153,8 @@ impl Instance { /// Rebooting to Running to Stopping to Stopped. pub async fn put_state( &self, - tx: oneshot::Sender>, - state: InstanceStateRequested, + tx: oneshot::Sender>, + state: VmmStateRequested, ) -> Result<(), Error> { self.tx .send(InstanceRequest::PutState { state, tx }) @@ -1166,7 +1167,7 @@ impl Instance { /// immediately transitions the instance to the Destroyed state. pub async fn terminate( &self, - tx: oneshot::Sender>, + tx: oneshot::Sender>, mark_failed: bool, ) -> Result<(), Error> { self.tx @@ -1224,7 +1225,7 @@ impl InstanceRunner { async fn request_zone_bundle( &self, ) -> Result { - let name = propolis_zone_name(self.propolis_id()); + let name = propolis_zone_name(&self.propolis_id); match &self.running_state { None => Err(BundleError::Unavailable { name }), Some(RunningState { ref running_zone, .. }) => { @@ -1242,7 +1243,7 @@ impl InstanceRunner { run_state.running_zone.root_zpool().map(|p| p.clone()) } - fn current_state(&self) -> SledInstanceState { + fn current_state(&self) -> SledVmmState { self.state.sled_instance_state() } @@ -1300,19 +1301,19 @@ impl InstanceRunner { async fn put_state( &mut self, - state: InstanceStateRequested, - ) -> Result { + state: VmmStateRequested, + ) -> Result { use propolis_client::types::InstanceStateRequested as PropolisRequest; let (propolis_state, next_published) = match state { - InstanceStateRequested::MigrationTarget(migration_params) => { + VmmStateRequested::MigrationTarget(migration_params) => { self.propolis_ensure(Some(migration_params)).await?; (None, None) } - InstanceStateRequested::Running => { + VmmStateRequested::Running => { self.propolis_ensure(None).await?; (Some(PropolisRequest::Run), None) } - InstanceStateRequested::Stopped => { + VmmStateRequested::Stopped => { // If the instance has not started yet, unregister it // immediately. Since there is no Propolis to push updates when // this happens, generate an instance record bearing the @@ -1328,9 +1329,9 @@ impl InstanceRunner { ) } } - InstanceStateRequested::Reboot => { + VmmStateRequested::Reboot => { if self.running_state.is_none() { - return Err(Error::InstanceNotRunning(self.id())); + return Err(Error::VmNotRunning(self.propolis_id)); } ( Some(PropolisRequest::Reboot), @@ -1379,7 +1380,7 @@ impl InstanceRunner { // Create a zone for the propolis instance, using the previously // configured VNICs. - let zname = propolis_zone_name(self.propolis_id()); + let zname = propolis_zone_name(&self.propolis_id); let mut rng = rand::rngs::StdRng::from_entropy(); let latest_disks = self .storage @@ -1399,7 +1400,7 @@ impl InstanceRunner { .with_zone_root_path(root) .with_zone_image_paths(&["/opt/oxide".into()]) .with_zone_type("propolis-server") - .with_unique_name(self.propolis_id().into_untyped_uuid()) + .with_unique_name(self.propolis_id.into_untyped_uuid()) .with_datasets(&[]) .with_filesystems(&[]) .with_data_links(&[]) @@ -1483,7 +1484,7 @@ impl InstanceRunner { Ok(PropolisSetup { client, running_zone }) } - async fn terminate(&mut self, mark_failed: bool) -> SledInstanceState { + async fn terminate(&mut self, mark_failed: bool) -> SledVmmState { self.terminate_inner().await; self.state.terminate_rudely(mark_failed); @@ -1508,9 +1509,7 @@ impl InstanceRunner { Ok(()) } else { - Err(Error::InstanceNotRunning(InstanceUuid::from_untyped_uuid( - self.properties.id, - ))) + Err(Error::VmNotRunning(self.propolis_id)) } } @@ -1604,7 +1603,7 @@ mod tests { enum ReceivedInstanceState { #[default] None, - InstancePut(SledInstanceState), + InstancePut(SledVmmState), } struct NexusServer { @@ -1614,8 +1613,8 @@ mod tests { impl FakeNexusServer for NexusServer { fn cpapi_instances_put( &self, - _instance_id: Uuid, - new_runtime_state: SledInstanceState, + _propolis_id: PropolisUuid, + new_runtime_state: SledVmmState, ) -> Result<(), omicron_common::api::external::Error> { self.observed_runtime_state .send(ReceivedInstanceState::InstancePut(new_runtime_state)) @@ -1760,7 +1759,7 @@ mod tests { let id = InstanceUuid::new_v4(); let propolis_id = PropolisUuid::from_untyped_uuid(PROPOLIS_ID); - let ticket = InstanceTicket::new_without_manager_for_test(id); + let ticket = InstanceTicket::new_without_manager_for_test(propolis_id); let initial_state = fake_instance_initial_state(propolis_addr); @@ -1917,7 +1916,7 @@ mod tests { // pretending we're InstanceManager::ensure_state, start our "instance" // (backed by fakes and propolis_mock_server) - inst.put_state(put_tx, InstanceStateRequested::Running) + inst.put_state(put_tx, VmmStateRequested::Running) .await .expect("failed to send Instance::put_state"); @@ -2011,7 +2010,7 @@ mod tests { // pretending we're InstanceManager::ensure_state, try in vain to start // our "instance", but no propolis server is running - inst.put_state(put_tx, InstanceStateRequested::Running) + inst.put_state(put_tx, VmmStateRequested::Running) .await .expect("failed to send Instance::put_state"); @@ -2025,7 +2024,7 @@ mod tests { .await .expect_err("*should've* timed out waiting for Instance::put_state, but didn't?"); - if let ReceivedInstanceState::InstancePut(SledInstanceState { + if let ReceivedInstanceState::InstancePut(SledVmmState { vmm_state: VmmRuntimeState { state: VmmState::Running, .. }, .. }) = state_rx.borrow().to_owned() @@ -2118,7 +2117,7 @@ mod tests { // pretending we're InstanceManager::ensure_state, try in vain to start // our "instance", but the zone never finishes installing - inst.put_state(put_tx, InstanceStateRequested::Running) + inst.put_state(put_tx, VmmStateRequested::Running) .await .expect("failed to send Instance::put_state"); @@ -2133,7 +2132,7 @@ mod tests { .expect_err("*should've* timed out waiting for Instance::put_state, but didn't?"); debug!(log, "Zone-boot timeout awaited"); - if let ReceivedInstanceState::InstancePut(SledInstanceState { + if let ReceivedInstanceState::InstancePut(SledVmmState { vmm_state: VmmRuntimeState { state: VmmState::Running, .. }, .. }) = state_rx.borrow().to_owned() @@ -2256,7 +2255,7 @@ mod tests { .await .unwrap(); - mgr.ensure_state(instance_id, InstanceStateRequested::Running) + mgr.ensure_state(propolis_id, VmmStateRequested::Running) .await .unwrap(); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 63164ed290..24be8be89f 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -4,13 +4,13 @@ //! API for controlling multiple instances on a sled. -use crate::instance::propolis_zone_name; use crate::instance::Instance; use crate::metrics::MetricsRequestQueue; use crate::nexus::NexusClient; use crate::vmm_reservoir::VmmReservoirManagerHandle; use crate::zone_bundle::BundleError; use crate::zone_bundle::ZoneBundler; +use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; use omicron_common::api::external::ByteCount; use anyhow::anyhow; @@ -20,7 +20,7 @@ use illumos_utils::opte::PortManager; use illumos_utils::running_zone::ZoneBuilderFactory; use omicron_common::api::external::Generation; use omicron_common::api::internal::nexus::InstanceRuntimeState; -use omicron_common::api::internal::nexus::SledInstanceState; +use omicron_common::api::internal::nexus::SledVmmState; use omicron_common::api::internal::nexus::VmmRuntimeState; use omicron_common::api::internal::shared::SledIdentifiers; use omicron_uuid_kinds::InstanceUuid; @@ -44,8 +44,8 @@ pub enum Error { #[error("Instance error: {0}")] Instance(#[from] crate::instance::Error), - #[error("No such instance ID: {0}")] - NoSuchInstance(InstanceUuid), + #[error("VMM with ID {0} not found")] + NoSuchVmm(PropolisUuid), #[error("OPTE port management error: {0}")] Opte(#[from] illumos_utils::opte::Error), @@ -117,7 +117,7 @@ impl InstanceManager { terminate_tx, terminate_rx, nexus_client, - instances: BTreeMap::new(), + jobs: BTreeMap::new(), vnic_allocator: VnicAllocator::new("Instance", etherstub), port_manager, storage_generation: None, @@ -150,7 +150,7 @@ impl InstanceManager { propolis_addr: SocketAddr, sled_identifiers: SledIdentifiers, metadata: InstanceMetadata, - ) -> Result { + ) -> Result { let (tx, rx) = oneshot::channel(); self.inner .tx @@ -172,13 +172,13 @@ impl InstanceManager { pub async fn ensure_unregistered( &self, - instance_id: InstanceUuid, - ) -> Result { + propolis_id: PropolisUuid, + ) -> Result { let (tx, rx) = oneshot::channel(); self.inner .tx .send(InstanceManagerRequest::EnsureUnregistered { - instance_id, + propolis_id, tx, }) .await @@ -188,14 +188,14 @@ impl InstanceManager { pub async fn ensure_state( &self, - instance_id: InstanceUuid, - target: InstanceStateRequested, - ) -> Result { + propolis_id: PropolisUuid, + target: VmmStateRequested, + ) -> Result { let (tx, rx) = oneshot::channel(); self.inner .tx .send(InstanceManagerRequest::EnsureState { - instance_id, + propolis_id, target, tx, }) @@ -206,31 +206,32 @@ impl InstanceManager { // these may involve a long-running zone creation, so avoid HTTP // request timeouts by decoupling the response // (see InstanceRunner::put_state) - InstanceStateRequested::MigrationTarget(_) - | InstanceStateRequested::Running => { + VmmStateRequested::MigrationTarget(_) + | VmmStateRequested::Running => { // We don't want the sending side of the channel to see an // error if we drop rx without awaiting it. // Since we don't care about the response here, we spawn rx // into a task which will await it for us in the background. tokio::spawn(rx); - Ok(InstancePutStateResponse { updated_runtime: None }) + Ok(VmmPutStateResponse { updated_runtime: None }) + } + VmmStateRequested::Stopped | VmmStateRequested::Reboot => { + rx.await? } - InstanceStateRequested::Stopped - | InstanceStateRequested::Reboot => rx.await?, } } - pub async fn instance_issue_disk_snapshot_request( + pub async fn issue_disk_snapshot_request( &self, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, disk_id: Uuid, snapshot_id: Uuid, ) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); self.inner .tx - .send(InstanceManagerRequest::InstanceIssueDiskSnapshot { - instance_id, + .send(InstanceManagerRequest::IssueDiskSnapshot { + propolis_id, disk_id, snapshot_id, tx, @@ -259,14 +260,14 @@ impl InstanceManager { pub async fn add_external_ip( &self, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, ip: &InstanceExternalIpBody, ) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); self.inner .tx - .send(InstanceManagerRequest::InstanceAddExternalIp { - instance_id, + .send(InstanceManagerRequest::AddExternalIp { + propolis_id, ip: *ip, tx, }) @@ -277,14 +278,14 @@ impl InstanceManager { pub async fn delete_external_ip( &self, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, ip: &InstanceExternalIpBody, ) -> Result<(), Error> { let (tx, rx) = oneshot::channel(); self.inner .tx - .send(InstanceManagerRequest::InstanceDeleteExternalIp { - instance_id, + .send(InstanceManagerRequest::DeleteExternalIp { + propolis_id, ip: *ip, tx, }) @@ -300,12 +301,12 @@ impl InstanceManager { pub async fn get_instance_state( &self, - instance_id: InstanceUuid, - ) -> Result { + propolis_id: PropolisUuid, + ) -> Result { let (tx, rx) = oneshot::channel(); self.inner .tx - .send(InstanceManagerRequest::GetState { instance_id, tx }) + .send(InstanceManagerRequest::GetState { propolis_id, tx }) .await .map_err(|_| Error::FailedSendInstanceManagerClosed)?; rx.await? @@ -351,20 +352,20 @@ enum InstanceManagerRequest { // reasonable choice... sled_identifiers: Box, metadata: InstanceMetadata, - tx: oneshot::Sender>, + tx: oneshot::Sender>, }, EnsureUnregistered { - instance_id: InstanceUuid, - tx: oneshot::Sender>, + propolis_id: PropolisUuid, + tx: oneshot::Sender>, }, EnsureState { - instance_id: InstanceUuid, - target: InstanceStateRequested, - tx: oneshot::Sender>, + propolis_id: PropolisUuid, + target: VmmStateRequested, + tx: oneshot::Sender>, }, - InstanceIssueDiskSnapshot { - instance_id: InstanceUuid, + IssueDiskSnapshot { + propolis_id: PropolisUuid, disk_id: Uuid, snapshot_id: Uuid, tx: oneshot::Sender>, @@ -373,19 +374,19 @@ enum InstanceManagerRequest { name: String, tx: oneshot::Sender>, }, - InstanceAddExternalIp { - instance_id: InstanceUuid, + AddExternalIp { + propolis_id: PropolisUuid, ip: InstanceExternalIpBody, tx: oneshot::Sender>, }, - InstanceDeleteExternalIp { - instance_id: InstanceUuid, + DeleteExternalIp { + propolis_id: PropolisUuid, ip: InstanceExternalIpBody, tx: oneshot::Sender>, }, GetState { - instance_id: InstanceUuid, - tx: oneshot::Sender>, + propolis_id: PropolisUuid, + tx: oneshot::Sender>, }, OnlyUseDisks { disks: AllDisks, @@ -396,7 +397,7 @@ enum InstanceManagerRequest { // Requests that the instance manager stop processing information about a // particular instance. struct InstanceDeregisterRequest { - id: InstanceUuid, + id: PropolisUuid, } struct InstanceManagerRunner { @@ -422,8 +423,8 @@ struct InstanceManagerRunner { // TODO: If we held an object representing an enum of "Created OR Running" // instance, we could avoid the methods within "instance.rs" that panic // if the Propolis client hasn't been initialized. - /// A mapping from a Sled Agent "Instance ID" to ("Propolis ID", [Instance]). - instances: BTreeMap, + /// A mapping from a Propolis ID to the [Instance] that Propolis incarnates. + jobs: BTreeMap, vnic_allocator: VnicAllocator, port_manager: PortManager, @@ -451,7 +452,7 @@ impl InstanceManagerRunner { request = self.terminate_rx.recv() => { match request { Some(request) => { - self.instances.remove(&request.id); + self.jobs.remove(&request.id); }, None => { warn!(self.log, "InstanceManager's 'instance terminate' channel closed; shutting down"); @@ -484,31 +485,31 @@ impl InstanceManagerRunner { metadata ).await).map_err(|_| Error::FailedSendClientClosed) }, - Some(EnsureUnregistered { instance_id, tx }) => { - self.ensure_unregistered(tx, instance_id).await + Some(EnsureUnregistered { propolis_id, tx }) => { + self.ensure_unregistered(tx, propolis_id).await }, - Some(EnsureState { instance_id, target, tx }) => { - self.ensure_state(tx, instance_id, target).await + Some(EnsureState { propolis_id, target, tx }) => { + self.ensure_state(tx, propolis_id, target).await }, - Some(InstanceIssueDiskSnapshot { instance_id, disk_id, snapshot_id, tx }) => { - self.instance_issue_disk_snapshot_request(tx, instance_id, disk_id, snapshot_id).await + Some(IssueDiskSnapshot { propolis_id, disk_id, snapshot_id, tx }) => { + self.issue_disk_snapshot_request(tx, propolis_id, disk_id, snapshot_id).await }, Some(CreateZoneBundle { name, tx }) => { self.create_zone_bundle(tx, &name).await.map_err(Error::from) }, - Some(InstanceAddExternalIp { instance_id, ip, tx }) => { - self.add_external_ip(tx, instance_id, &ip).await + Some(AddExternalIp { propolis_id, ip, tx }) => { + self.add_external_ip(tx, propolis_id, &ip).await }, - Some(InstanceDeleteExternalIp { instance_id, ip, tx }) => { - self.delete_external_ip(tx, instance_id, &ip).await + Some(DeleteExternalIp { propolis_id, ip, tx }) => { + self.delete_external_ip(tx, propolis_id, &ip).await }, - Some(GetState { instance_id, tx }) => { + Some(GetState { propolis_id, tx }) => { // TODO(eliza): it could potentially be nice to // refactor this to use `tokio::sync::watch`, rather // than having to force `GetState` requests to // serialize with the requests that actually update // the state... - self.get_instance_state(tx, instance_id).await + self.get_instance_state(tx, propolis_id).await }, Some(OnlyUseDisks { disks, tx } ) => { self.use_only_these_disks(disks).await; @@ -533,8 +534,8 @@ impl InstanceManagerRunner { } } - fn get_instance(&self, instance_id: InstanceUuid) -> Option<&Instance> { - self.instances.get(&instance_id).map(|(_id, v)| v) + fn get_propolis(&self, propolis_id: PropolisUuid) -> Option<&Instance> { + self.jobs.get(&propolis_id) } /// Ensures that the instance manager contains a registered instance with @@ -565,7 +566,7 @@ impl InstanceManagerRunner { propolis_addr: SocketAddr, sled_identifiers: SledIdentifiers, metadata: InstanceMetadata, - ) -> Result { + ) -> Result { info!( &self.log, "ensuring instance is registered"; @@ -579,17 +580,16 @@ impl InstanceManagerRunner { ); let instance = { - if let Some((existing_propolis_id, existing_instance)) = - self.instances.get(&instance_id) - { - if propolis_id != *existing_propolis_id { + if let Some(existing_instance) = self.jobs.get(&propolis_id) { + if instance_id != existing_instance.id() { info!(&self.log, - "instance already registered with another Propolis ID"; - "instance_id" => %instance_id, - "existing_propolis_id" => %*existing_propolis_id); + "Propolis ID already used by another instance"; + "propolis_id" => %propolis_id, + "existing_instanceId" => %existing_instance.id()); + return Err(Error::Instance( - crate::instance::Error::InstanceAlreadyRegistered( - *existing_propolis_id, + crate::instance::Error::PropolisAlreadyRegistered( + propolis_id, ), )); } else { @@ -602,11 +602,16 @@ impl InstanceManagerRunner { } else { info!(&self.log, "registering new instance"; - "instance_id" => ?instance_id); - let instance_log = - self.log.new(o!("instance_id" => format!("{instance_id}"))); + "instance_id" => %instance_id, + "propolis_id" => %propolis_id); + + let instance_log = self.log.new(o!( + "instance_id" => instance_id.to_string(), + "propolis_id" => propolis_id.to_string(), + )); + let ticket = - InstanceTicket::new(instance_id, self.terminate_tx.clone()); + InstanceTicket::new(propolis_id, self.terminate_tx.clone()); let services = InstanceManagerServices { nexus_client: self.nexus_client.clone(), @@ -635,27 +640,26 @@ impl InstanceManagerRunner { sled_identifiers, metadata, )?; - let _old = - self.instances.insert(instance_id, (propolis_id, instance)); + let _old = self.jobs.insert(propolis_id, instance); assert!(_old.is_none()); - &self.instances.get(&instance_id).unwrap().1 + &self.jobs.get(&propolis_id).unwrap() } }; Ok(instance.current_state().await?) } - /// Idempotently ensures the instance is not registered with this instance - /// manager. If the instance exists and has a running Propolis, that - /// Propolis is rudely terminated. + /// Idempotently ensures this VM is not registered with this instance + /// manager. If this Propolis job is registered and has a running zone, the + /// zone is rudely terminated. async fn ensure_unregistered( &mut self, - tx: oneshot::Sender>, - instance_id: InstanceUuid, + tx: oneshot::Sender>, + propolis_id: PropolisUuid, ) -> Result<(), Error> { // If the instance does not exist, we response immediately. - let Some(instance) = self.get_instance(instance_id) else { - tx.send(Ok(InstanceUnregisterResponse { updated_runtime: None })) + let Some(instance) = self.get_propolis(propolis_id) else { + tx.send(Ok(VmmUnregisterResponse { updated_runtime: None })) .map_err(|_| Error::FailedSendClientClosed)?; return Ok(()); }; @@ -667,15 +671,15 @@ impl InstanceManagerRunner { Ok(()) } - /// Idempotently attempts to drive the supplied instance into the supplied + /// Idempotently attempts to drive the supplied Propolis into the supplied /// runtime state. async fn ensure_state( &mut self, - tx: oneshot::Sender>, - instance_id: InstanceUuid, - target: InstanceStateRequested, + tx: oneshot::Sender>, + propolis_id: PropolisUuid, + target: VmmStateRequested, ) -> Result<(), Error> { - let Some(instance) = self.get_instance(instance_id) else { + let Some(instance) = self.get_propolis(propolis_id) else { match target { // If the instance isn't registered, then by definition it // isn't running here. Allow requests to stop or destroy the @@ -685,14 +689,12 @@ impl InstanceManagerRunner { // Propolis handled it, sled agent unregistered the // instance, and only then did a second stop request // arrive. - InstanceStateRequested::Stopped => { - tx.send(Ok(InstancePutStateResponse { - updated_runtime: None, - })) - .map_err(|_| Error::FailedSendClientClosed)?; + VmmStateRequested::Stopped => { + tx.send(Ok(VmmPutStateResponse { updated_runtime: None })) + .map_err(|_| Error::FailedSendClientClosed)?; } _ => { - tx.send(Err(Error::NoSuchInstance(instance_id))) + tx.send(Err(Error::NoSuchVmm(propolis_id))) .map_err(|_| Error::FailedSendClientClosed)?; } } @@ -702,20 +704,15 @@ impl InstanceManagerRunner { Ok(()) } - async fn instance_issue_disk_snapshot_request( + async fn issue_disk_snapshot_request( &self, tx: oneshot::Sender>, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, disk_id: Uuid, snapshot_id: Uuid, ) -> Result<(), Error> { - let instance = { - let (_, instance) = self - .instances - .get(&instance_id) - .ok_or(Error::NoSuchInstance(instance_id))?; - instance - }; + let instance = + self.jobs.get(&propolis_id).ok_or(Error::NoSuchVmm(propolis_id))?; instance .issue_snapshot_request(tx, disk_id, snapshot_id) @@ -729,11 +726,19 @@ impl InstanceManagerRunner { tx: oneshot::Sender>, name: &str, ) -> Result<(), BundleError> { - let Some((_propolis_id, instance)) = - self.instances.values().find(|(propolis_id, _instance)| { - name == propolis_zone_name(propolis_id) - }) - else { + // A well-formed Propolis zone name must consist of + // `PROPOLIS_ZONE_PREFIX` and the Propolis ID. If the prefix is not + // present or the Propolis ID portion of the supplied zone name isn't + // parseable as a UUID, there is no Propolis zone with the specified + // name to capture into a bundle, so return a `NoSuchZone` error. + let vmm_id: PropolisUuid = name + .strip_prefix(PROPOLIS_ZONE_PREFIX) + .and_then(|uuid_str| uuid_str.parse::().ok()) + .ok_or_else(|| BundleError::NoSuchZone { + name: name.to_string(), + })?; + + let Some(instance) = self.jobs.get(&vmm_id) else { return Err(BundleError::NoSuchZone { name: name.to_string() }); }; instance.request_zone_bundle(tx).await @@ -742,11 +747,11 @@ impl InstanceManagerRunner { async fn add_external_ip( &self, tx: oneshot::Sender>, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, ip: &InstanceExternalIpBody, ) -> Result<(), Error> { - let Some(instance) = self.get_instance(instance_id) else { - return Err(Error::NoSuchInstance(instance_id)); + let Some(instance) = self.get_propolis(propolis_id) else { + return Err(Error::NoSuchVmm(propolis_id)); }; instance.add_external_ip(tx, ip).await?; Ok(()) @@ -755,11 +760,11 @@ impl InstanceManagerRunner { async fn delete_external_ip( &self, tx: oneshot::Sender>, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, ip: &InstanceExternalIpBody, ) -> Result<(), Error> { - let Some(instance) = self.get_instance(instance_id) else { - return Err(Error::NoSuchInstance(instance_id)); + let Some(instance) = self.get_propolis(propolis_id) else { + return Err(Error::NoSuchVmm(propolis_id)); }; instance.delete_external_ip(tx, ip).await?; @@ -768,12 +773,12 @@ impl InstanceManagerRunner { async fn get_instance_state( &self, - tx: oneshot::Sender>, - instance_id: InstanceUuid, + tx: oneshot::Sender>, + propolis_id: PropolisUuid, ) -> Result<(), Error> { - let Some(instance) = self.get_instance(instance_id) else { + let Some(instance) = self.get_propolis(propolis_id) else { return tx - .send(Err(Error::NoSuchInstance(instance_id))) + .send(Err(Error::NoSuchVmm(propolis_id))) .map_err(|_| Error::FailedSendClientClosed); }; @@ -801,7 +806,7 @@ impl InstanceManagerRunner { let u2_set: HashSet<_> = disks.all_u2_zpools().into_iter().collect(); let mut to_remove = vec![]; - for (id, (_, instance)) in self.instances.iter() { + for (id, instance) in self.jobs.iter() { // If we can read the filesystem pool, consider it. Otherwise, move // on, to prevent blocking the cleanup of other instances. let Ok(Some(filesystem_pool)) = @@ -817,7 +822,7 @@ impl InstanceManagerRunner { for id in to_remove { info!(self.log, "use_only_these_disks: Removing instance"; "instance_id" => ?id); - if let Some((_, instance)) = self.instances.remove(&id) { + if let Some(instance) = self.jobs.remove(&id) { let (tx, rx) = oneshot::channel(); let mark_failed = true; if let Err(e) = instance.terminate(tx, mark_failed).await { @@ -835,22 +840,22 @@ impl InstanceManagerRunner { /// Represents membership of an instance in the [`InstanceManager`]. pub struct InstanceTicket { - id: InstanceUuid, + id: PropolisUuid, terminate_tx: Option>, } impl InstanceTicket { - // Creates a new instance ticket for instance "id" to be removed - // from the manger on destruction. + // Creates a new instance ticket for the Propolis job with the supplied `id` + // to be removed from the manager on destruction. fn new( - id: InstanceUuid, + id: PropolisUuid, terminate_tx: mpsc::UnboundedSender, ) -> Self { InstanceTicket { id, terminate_tx: Some(terminate_tx) } } #[cfg(all(test, target_os = "illumos"))] - pub(crate) fn new_without_manager_for_test(id: InstanceUuid) -> Self { + pub(crate) fn new_without_manager_for_test(id: PropolisUuid) -> Self { Self { id, terminate_tx: None } } diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 419e897d75..de0b086752 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -3,9 +3,8 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use nexus_sled_agent_shared::inventory::{OmicronZoneConfig, OmicronZoneType}; +use omicron_common::disk::{DatasetKind, DatasetName}; pub use sled_hardware::DendriteAsic; -use sled_storage::dataset::DatasetName; -use sled_storage::dataset::DatasetType; use std::net::SocketAddrV6; /// Extension trait for `OmicronZoneConfig`. @@ -49,25 +48,25 @@ pub(crate) trait OmicronZoneTypeExt { | OmicronZoneType::Oximeter { .. } | OmicronZoneType::CruciblePantry { .. } => None, OmicronZoneType::Clickhouse { dataset, address, .. } => { - Some((dataset, DatasetType::Clickhouse, address)) + Some((dataset, DatasetKind::Clickhouse, address)) } OmicronZoneType::ClickhouseKeeper { dataset, address, .. } => { - Some((dataset, DatasetType::ClickhouseKeeper, address)) + Some((dataset, DatasetKind::ClickhouseKeeper, address)) } OmicronZoneType::ClickhouseServer { dataset, address, .. } => { - Some((dataset, DatasetType::ClickhouseServer, address)) + Some((dataset, DatasetKind::ClickhouseServer, address)) } OmicronZoneType::CockroachDb { dataset, address, .. } => { - Some((dataset, DatasetType::CockroachDb, address)) + Some((dataset, DatasetKind::Cockroach, address)) } OmicronZoneType::Crucible { dataset, address, .. } => { - Some((dataset, DatasetType::Crucible, address)) + Some((dataset, DatasetKind::Crucible, address)) } OmicronZoneType::ExternalDns { dataset, http_address, .. } => { - Some((dataset, DatasetType::ExternalDns, http_address)) + Some((dataset, DatasetKind::ExternalDns, http_address)) } OmicronZoneType::InternalDns { dataset, http_address, .. } => { - Some((dataset, DatasetType::InternalDns, http_address)) + Some((dataset, DatasetKind::InternalDns, http_address)) } }?; diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index a376096a87..7ca2b295a0 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -32,12 +32,13 @@ use omicron_common::backoff::{ retry_notify_ext, retry_policy_internal_service_aggressive, BackoffError, }; use omicron_common::disk::{ - DiskVariant, OmicronPhysicalDiskConfig, OmicronPhysicalDisksConfig, + DatasetKind, DatasetName, DiskVariant, OmicronPhysicalDiskConfig, + OmicronPhysicalDisksConfig, }; use omicron_common::ledger::{self, Ledger, Ledgerable}; use omicron_common::policy::{ - BOUNDARY_NTP_REDUNDANCY, COCKROACHDB_REDUNDANCY, DNS_REDUNDANCY, - MAX_DNS_REDUNDANCY, NEXUS_REDUNDANCY, + BOUNDARY_NTP_REDUNDANCY, COCKROACHDB_REDUNDANCY, INTERNAL_DNS_REDUNDANCY, + MAX_INTERNAL_DNS_REDUNDANCY, NEXUS_REDUNDANCY, }; use omicron_uuid_kinds::{ ExternalIpUuid, GenericUuid, OmicronZoneUuid, SledUuid, ZpoolUuid, @@ -50,7 +51,7 @@ use sled_agent_client::{ }; use sled_agent_types::rack_init::RackInitializeRequest as Config; use sled_agent_types::sled::StartSledAgentRequest; -use sled_storage::dataset::{DatasetName, DatasetType, CONFIG_DATASET}; +use sled_storage::dataset::CONFIG_DATASET; use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -469,9 +470,11 @@ impl Plan { // Provision internal DNS zones, striping across Sleds. let reserved_rack_subnet = ReservedRackSubnet::new(config.az_subnet()); - static_assertions::const_assert!(DNS_REDUNDANCY <= MAX_DNS_REDUNDANCY,); + static_assertions::const_assert!( + INTERNAL_DNS_REDUNDANCY <= MAX_INTERNAL_DNS_REDUNDANCY + ); let dns_subnets = - &reserved_rack_subnet.get_dns_subnets()[0..DNS_REDUNDANCY]; + &reserved_rack_subnet.get_dns_subnets()[0..INTERNAL_DNS_REDUNDANCY]; let rack_dns_servers = dns_subnets .into_iter() .map(|dns_subnet| dns_subnet.dns_address().into()) @@ -497,7 +500,7 @@ impl Plan { ) .unwrap(); let dataset_name = - sled.alloc_dataset_from_u2s(DatasetType::InternalDns)?; + sled.alloc_dataset_from_u2s(DatasetKind::InternalDns)?; let filesystem_pool = Some(dataset_name.pool().clone()); sled.request.zones.push(BlueprintZoneConfig { @@ -539,7 +542,7 @@ impl Plan { ) .unwrap(); let dataset_name = - sled.alloc_dataset_from_u2s(DatasetType::CockroachDb)?; + sled.alloc_dataset_from_u2s(DatasetKind::Cockroach)?; let filesystem_pool = Some(dataset_name.pool().clone()); sled.request.zones.push(BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, @@ -587,7 +590,7 @@ impl Plan { let dns_address = from_sockaddr_to_external_floating_addr( SocketAddr::new(external_ip, dns_port), ); - let dataset_kind = DatasetType::ExternalDns; + let dataset_kind = DatasetKind::ExternalDns; let dataset_name = sled.alloc_dataset_from_u2s(dataset_kind)?; let filesystem_pool = Some(dataset_name.pool().clone()); @@ -705,7 +708,7 @@ impl Plan { }; let id = OmicronZoneUuid::new_v4(); let ip = sled.addr_alloc.next().expect("Not enough addrs"); - let port = omicron_common::address::CLICKHOUSE_PORT; + let port = omicron_common::address::CLICKHOUSE_HTTP_PORT; let address = SocketAddrV6::new(ip, port, 0, 0); dns_builder .host_zone_with_one_backend( @@ -716,7 +719,7 @@ impl Plan { ) .unwrap(); let dataset_name = - sled.alloc_dataset_from_u2s(DatasetType::Clickhouse)?; + sled.alloc_dataset_from_u2s(DatasetKind::Clickhouse)?; let filesystem_pool = Some(dataset_name.pool().clone()); sled.request.zones.push(BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, @@ -748,7 +751,7 @@ impl Plan { let ip = sled.addr_alloc.next().expect("Not enough addrs"); // TODO: This may need to be a different port if/when to have single node // and replicated running side by side as per stage 1 of RFD 468. - let port = omicron_common::address::CLICKHOUSE_PORT; + let port = omicron_common::address::CLICKHOUSE_HTTP_PORT; let address = SocketAddrV6::new(ip, port, 0, 0); dns_builder .host_zone_with_one_backend( @@ -759,7 +762,7 @@ impl Plan { ) .unwrap(); let dataset_name = - sled.alloc_dataset_from_u2s(DatasetType::ClickhouseServer)?; + sled.alloc_dataset_from_u2s(DatasetKind::ClickhouseServer)?; let filesystem_pool = Some(dataset_name.pool().clone()); sled.request.zones.push(BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, @@ -789,7 +792,7 @@ impl Plan { }; let id = OmicronZoneUuid::new_v4(); let ip = sled.addr_alloc.next().expect("Not enough addrs"); - let port = omicron_common::address::CLICKHOUSE_KEEPER_PORT; + let port = omicron_common::address::CLICKHOUSE_KEEPER_TCP_PORT; let address = SocketAddrV6::new(ip, port, 0, 0); dns_builder .host_zone_with_one_backend( @@ -800,7 +803,7 @@ impl Plan { ) .unwrap(); let dataset_name = - sled.alloc_dataset_from_u2s(DatasetType::ClickhouseKeeper)?; + sled.alloc_dataset_from_u2s(DatasetKind::ClickhouseKeeper)?; let filesystem_pool = Some(dataset_name.pool().clone()); sled.request.zones.push(BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, @@ -1034,7 +1037,7 @@ pub struct SledInfo { u2_zpools: Vec, /// spreads components across a Sled's zpools u2_zpool_allocators: - HashMap + Send + Sync>>, + HashMap + Send + Sync>>, /// whether this Sled is a scrimlet is_scrimlet: bool, /// allocator for addresses in this Sled's subnet @@ -1075,7 +1078,7 @@ impl SledInfo { /// this Sled fn alloc_dataset_from_u2s( &mut self, - kind: DatasetType, + kind: DatasetKind, ) -> Result { // We have two goals here: // diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 22cbb62f70..7677dfbd8a 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -66,8 +66,8 @@ use nexus_sled_agent_shared::inventory::{ OmicronZoneConfig, OmicronZoneType, OmicronZonesConfig, ZoneKind, }; use omicron_common::address::CLICKHOUSE_ADMIN_PORT; -use omicron_common::address::CLICKHOUSE_KEEPER_PORT; -use omicron_common::address::CLICKHOUSE_PORT; +use omicron_common::address::CLICKHOUSE_HTTP_PORT; +use omicron_common::address::CLICKHOUSE_KEEPER_TCP_PORT; use omicron_common::address::COCKROACH_PORT; use omicron_common::address::CRUCIBLE_PANTRY_PORT; use omicron_common::address::CRUCIBLE_PORT; @@ -90,6 +90,7 @@ use omicron_common::api::internal::shared::{ use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, BackoffError, }; +use omicron_common::disk::{DatasetKind, DatasetName}; use omicron_common::ledger::{self, Ledger, Ledgerable}; use omicron_ddm_admin_client::{Client as DdmAdminClient, DdmError}; use once_cell::sync::OnceCell; @@ -103,9 +104,7 @@ use sled_hardware::underlay; use sled_hardware::SledMode; use sled_hardware_types::Baseboard; use sled_storage::config::MountConfig; -use sled_storage::dataset::{ - DatasetName, DatasetType, CONFIG_DATASET, INSTALL_DATASET, ZONE_DATASET, -}; +use sled_storage::dataset::{CONFIG_DATASET, INSTALL_DATASET, ZONE_DATASET}; use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::BTreeMap; @@ -1550,7 +1549,7 @@ impl ServiceManager { }; let listen_addr = *underlay_address; - let listen_port = &CLICKHOUSE_PORT.to_string(); + let listen_port = &CLICKHOUSE_HTTP_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( Some(&info.underlay_address), @@ -1574,9 +1573,11 @@ impl ServiceManager { .add_property_group(config), ); - let ch_address = - SocketAddr::new(IpAddr::V6(listen_addr), CLICKHOUSE_PORT) - .to_string(); + let ch_address = SocketAddr::new( + IpAddr::V6(listen_addr), + CLICKHOUSE_HTTP_PORT, + ) + .to_string(); let admin_address = SocketAddr::new( IpAddr::V6(listen_addr), @@ -1628,7 +1629,7 @@ impl ServiceManager { }; let listen_addr = *underlay_address; - let listen_port = CLICKHOUSE_PORT.to_string(); + let listen_port = CLICKHOUSE_HTTP_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( Some(&info.underlay_address), @@ -1653,9 +1654,11 @@ impl ServiceManager { .add_property_group(config), ); - let ch_address = - SocketAddr::new(IpAddr::V6(listen_addr), CLICKHOUSE_PORT) - .to_string(); + let ch_address = SocketAddr::new( + IpAddr::V6(listen_addr), + CLICKHOUSE_HTTP_PORT, + ) + .to_string(); let admin_address = SocketAddr::new( IpAddr::V6(listen_addr), @@ -1710,7 +1713,7 @@ impl ServiceManager { }; let listen_addr = *underlay_address; - let listen_port = &CLICKHOUSE_KEEPER_PORT.to_string(); + let listen_port = &CLICKHOUSE_KEEPER_TCP_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( Some(&info.underlay_address), @@ -1735,9 +1738,11 @@ impl ServiceManager { .add_property_group(config), ); - let ch_address = - SocketAddr::new(IpAddr::V6(listen_addr), CLICKHOUSE_PORT) - .to_string(); + let ch_address = SocketAddr::new( + IpAddr::V6(listen_addr), + CLICKHOUSE_HTTP_PORT, + ) + .to_string(); let admin_address = SocketAddr::new( IpAddr::V6(listen_addr), @@ -1875,7 +1880,7 @@ impl ServiceManager { let dataset_name = DatasetName::new( dataset.pool_name.clone(), - DatasetType::Crucible, + DatasetKind::Crucible, ) .full_name(); let uuid = &Uuid::new_v4().to_string(); diff --git a/sled-agent/src/sim/collection.rs b/sled-agent/src/sim/collection.rs index 6057d03f70..d75081f1e4 100644 --- a/sled-agent/src/sim/collection.rs +++ b/sled-agent/src/sim/collection.rs @@ -364,35 +364,6 @@ impl SimCollection { pub async fn contains_key(self: &Arc, id: &Uuid) -> bool { self.objects.lock().await.contains_key(id) } - - /// Iterates over all of the existing objects in the collection and, for any - /// that meet `condition`, asks to transition them into the supplied target - /// state. - /// - /// If any such transition fails, this routine short-circuits and does not - /// attempt to transition any other objects. - // - // TODO: It's likely more idiomatic to have an `iter_mut` routine that - // returns a struct that impls Iterator and yields &mut S references. The - // tricky bit is that the struct must hold the objects lock during the - // iteration. Figure out if there's a better way to arrange all this. - pub async fn sim_ensure_for_each_where( - self: &Arc, - condition: C, - target: &S::RequestedState, - ) -> Result<(), Error> - where - C: Fn(&S) -> bool, - { - let mut objects = self.objects.lock().await; - for o in objects.values_mut() { - if condition(&o.object) { - o.transition(target.clone())?; - } - } - - Ok(()) - } } impl SimCollection { @@ -421,30 +392,24 @@ mod test { use omicron_common::api::external::Error; use omicron_common::api::external::Generation; use omicron_common::api::internal::nexus::DiskRuntimeState; - use omicron_common::api::internal::nexus::SledInstanceState; + use omicron_common::api::internal::nexus::SledVmmState; use omicron_common::api::internal::nexus::VmmRuntimeState; use omicron_common::api::internal::nexus::VmmState; use omicron_test_utils::dev::test_setup_log; - use omicron_uuid_kinds::PropolisUuid; use sled_agent_types::disk::DiskStateRequested; - use sled_agent_types::instance::InstanceStateRequested; + use sled_agent_types::instance::VmmStateRequested; fn make_instance( logctx: &LogContext, ) -> (SimObject, Receiver<()>) { - let propolis_id = PropolisUuid::new_v4(); let vmm_state = VmmRuntimeState { state: VmmState::Starting, gen: Generation::new(), time_updated: Utc::now(), }; - let state = SledInstanceState { - vmm_state, - propolis_id, - migration_in: None, - migration_out: None, - }; + let state = + SledVmmState { vmm_state, migration_in: None, migration_out: None }; SimObject::new_simulated_auto(&state, logctx.log.new(o!())) } @@ -488,8 +453,7 @@ mod test { // Stopping an instance that was never started synchronously destroys // its VMM. let rprev = r1; - let dropped = - instance.transition(InstanceStateRequested::Stopped).unwrap(); + let dropped = instance.transition(VmmStateRequested::Stopped).unwrap(); assert!(dropped.is_none()); assert!(instance.object.desired().is_none()); let rnext = instance.object.current(); @@ -529,8 +493,7 @@ mod test { // simulated instance's state, but it does queue up a transition. let mut rprev = r1; assert!(rx.try_next().is_err()); - let dropped = - instance.transition(InstanceStateRequested::Running).unwrap(); + let dropped = instance.transition(VmmStateRequested::Running).unwrap(); assert!(dropped.is_none()); assert!(instance.object.desired().is_some()); assert!(rx.try_next().is_err()); @@ -562,8 +525,7 @@ mod test { // If we transition again to "Running", the process should complete // immediately. - let dropped = - instance.transition(InstanceStateRequested::Running).unwrap(); + let dropped = instance.transition(VmmStateRequested::Running).unwrap(); assert!(dropped.is_none()); assert!(instance.object.desired().is_none()); assert!(rx.try_next().is_err()); @@ -576,8 +538,7 @@ mod test { // If we go back to any stopped state, we go through the async process // again. assert!(rx.try_next().is_err()); - let dropped = - instance.transition(InstanceStateRequested::Stopped).unwrap(); + let dropped = instance.transition(VmmStateRequested::Stopped).unwrap(); assert!(dropped.is_none()); assert!(instance.object.desired().is_some()); let rnext = instance.object.current(); @@ -634,7 +595,7 @@ mod test { assert_eq!(r1.vmm_state.state, VmmState::Starting); assert_eq!(r1.vmm_state.gen, Generation::new()); assert!(instance - .transition(InstanceStateRequested::Running) + .transition(VmmStateRequested::Running) .unwrap() .is_none()); instance.transition_finish(); @@ -650,7 +611,7 @@ mod test { // Now reboot the instance. This is dispatched to Propolis, which will // move to the Rebooting state and then back to Running. assert!(instance - .transition(InstanceStateRequested::Reboot) + .transition(VmmStateRequested::Reboot) .unwrap() .is_none()); let (rprev, rnext) = (rnext, instance.object.current()); diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index e93bebad98..ac583a1a74 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -23,16 +23,17 @@ use dropshot::TypedBody; use nexus_sled_agent_shared::inventory::SledRole; use nexus_sled_agent_shared::inventory::{Inventory, OmicronZonesConfig}; use omicron_common::api::internal::nexus::DiskRuntimeState; -use omicron_common::api::internal::nexus::SledInstanceState; +use omicron_common::api::internal::nexus::SledVmmState; use omicron_common::api::internal::nexus::UpdateArtifactId; use omicron_common::api::internal::shared::SledIdentifiers; use omicron_common::api::internal::shared::VirtualNetworkInterfaceHost; use omicron_common::api::internal::shared::{ ResolvedVpcRouteSet, ResolvedVpcRouteState, SwitchPorts, }; +use omicron_common::disk::DatasetsConfig; +use omicron_common::disk::DatasetsManagementResult; use omicron_common::disk::DisksManagementResult; use omicron_common::disk::OmicronPhysicalDisksConfig; -use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; use sled_agent_api::*; use sled_agent_types::boot_disk::BootDiskOsWriteStatus; use sled_agent_types::boot_disk::BootDiskPathParams; @@ -44,9 +45,9 @@ use sled_agent_types::early_networking::EarlyNetworkConfig; use sled_agent_types::firewall_rules::VpcFirewallRulesEnsureBody; use sled_agent_types::instance::InstanceEnsureBody; use sled_agent_types::instance::InstanceExternalIpBody; -use sled_agent_types::instance::InstancePutStateBody; -use sled_agent_types::instance::InstancePutStateResponse; -use sled_agent_types::instance::InstanceUnregisterResponse; +use sled_agent_types::instance::VmmPutStateBody; +use sled_agent_types::instance::VmmPutStateResponse; +use sled_agent_types::instance::VmmUnregisterResponse; use sled_agent_types::sled::AddSledRequest; use sled_agent_types::time_sync::TimeSync; use sled_agent_types::zone_bundle::BundleUtilization; @@ -83,18 +84,18 @@ enum SledAgentSimImpl {} impl SledAgentApi for SledAgentSimImpl { type Context = Arc; - async fn instance_register( + async fn vmm_register( rqctx: RequestContext, - path_params: Path, + path_params: Path, body: TypedBody, - ) -> Result, HttpError> { + ) -> Result, HttpError> { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; + let propolis_id = path_params.into_inner().propolis_id; let body_args = body.into_inner(); Ok(HttpResponseOk( sa.instance_register( - instance_id, - body_args.propolis_id, + body_args.instance_id, + propolis_id, body_args.hardware, body_args.instance_runtime, body_args.vmm_runtime, @@ -104,58 +105,56 @@ impl SledAgentApi for SledAgentSimImpl { )) } - async fn instance_unregister( + async fn vmm_unregister( rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError> { + path_params: Path, + ) -> Result, HttpError> { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; - Ok(HttpResponseOk(sa.instance_unregister(instance_id).await?)) + let id = path_params.into_inner().propolis_id; + Ok(HttpResponseOk(sa.instance_unregister(id).await?)) } - async fn instance_put_state( + async fn vmm_put_state( rqctx: RequestContext, - path_params: Path, - body: TypedBody, - ) -> Result, HttpError> { + path_params: Path, + body: TypedBody, + ) -> Result, HttpError> { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; + let id = path_params.into_inner().propolis_id; let body_args = body.into_inner(); - Ok(HttpResponseOk( - sa.instance_ensure_state(instance_id, body_args.state).await?, - )) + Ok(HttpResponseOk(sa.instance_ensure_state(id, body_args.state).await?)) } - async fn instance_get_state( + async fn vmm_get_state( rqctx: RequestContext, - path_params: Path, - ) -> Result, HttpError> { + path_params: Path, + ) -> Result, HttpError> { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; - Ok(HttpResponseOk(sa.instance_get_state(instance_id).await?)) + let id = path_params.into_inner().propolis_id; + Ok(HttpResponseOk(sa.instance_get_state(id).await?)) } - async fn instance_put_external_ip( + async fn vmm_put_external_ip( rqctx: RequestContext, - path_params: Path, + path_params: Path, body: TypedBody, ) -> Result { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; + let id = path_params.into_inner().propolis_id; let body_args = body.into_inner(); - sa.instance_put_external_ip(instance_id, &body_args).await?; + sa.instance_put_external_ip(id, &body_args).await?; Ok(HttpResponseUpdatedNoContent()) } - async fn instance_delete_external_ip( + async fn vmm_delete_external_ip( rqctx: RequestContext, - path_params: Path, + path_params: Path, body: TypedBody, ) -> Result { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; + let id = path_params.into_inner().propolis_id; let body_args = body.into_inner(); - sa.instance_delete_external_ip(instance_id, &body_args).await?; + sa.instance_delete_external_ip(id, &body_args).await?; Ok(HttpResponseUpdatedNoContent()) } @@ -192,27 +191,25 @@ impl SledAgentApi for SledAgentSimImpl { Ok(HttpResponseUpdatedNoContent()) } - async fn instance_issue_disk_snapshot_request( + async fn vmm_issue_disk_snapshot_request( rqctx: RequestContext, - path_params: Path, - body: TypedBody, - ) -> Result< - HttpResponseOk, - HttpError, - > { + path_params: Path, + body: TypedBody, + ) -> Result, HttpError> + { let sa = rqctx.context(); let path_params = path_params.into_inner(); let body = body.into_inner(); sa.instance_issue_disk_snapshot_request( - InstanceUuid::from_untyped_uuid(path_params.instance_id), + path_params.propolis_id, path_params.disk_id, body.snapshot_id, ) .await .map_err(|e| HttpError::for_internal_error(e.to_string()))?; - Ok(HttpResponseOk(InstanceIssueDiskSnapshotRequestResponse { + Ok(HttpResponseOk(VmmIssueDiskSnapshotRequestResponse { snapshot_id: body.snapshot_id, })) } @@ -304,6 +301,23 @@ impl SledAgentApi for SledAgentSimImpl { )) } + async fn datasets_put( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError> { + let sa = rqctx.context(); + let body_args = body.into_inner(); + let result = sa.datasets_ensure(body_args).await?; + Ok(HttpResponseOk(result)) + } + + async fn datasets_get( + rqctx: RequestContext, + ) -> Result, HttpError> { + let sa = rqctx.context(); + Ok(HttpResponseOk(sa.datasets_config_list().await?)) + } + async fn omicron_physical_disks_put( rqctx: RequestContext, body: TypedBody, @@ -512,45 +526,44 @@ fn method_unimplemented() -> Result { #[endpoint { method = POST, - path = "/instances/{instance_id}/poke", + path = "/vmms/{propolis_id}/poke", }] async fn instance_poke_post( rqctx: RequestContext>, - path_params: Path, + path_params: Path, ) -> Result { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; - sa.instance_poke(instance_id, PokeMode::Drain).await; + let id = path_params.into_inner().propolis_id; + sa.vmm_poke(id, PokeMode::Drain).await; Ok(HttpResponseUpdatedNoContent()) } #[endpoint { method = POST, - path = "/instances/{instance_id}/poke-single-step", + path = "/vmms/{propolis_id}/poke-single-step", }] async fn instance_poke_single_step_post( rqctx: RequestContext>, - path_params: Path, + path_params: Path, ) -> Result { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; - sa.instance_poke(instance_id, PokeMode::SingleStep).await; + let id = path_params.into_inner().propolis_id; + sa.vmm_poke(id, PokeMode::SingleStep).await; Ok(HttpResponseUpdatedNoContent()) } #[endpoint { method = POST, - path = "/instances/{instance_id}/sim-migration-source", + path = "/vmms/{propolis_id}/sim-migration-source", }] async fn instance_post_sim_migration_source( rqctx: RequestContext>, - path_params: Path, + path_params: Path, body: TypedBody, ) -> Result { let sa = rqctx.context(); - let instance_id = path_params.into_inner().instance_id; - sa.instance_simulate_migration_source(instance_id, body.into_inner()) - .await?; + let id = path_params.into_inner().propolis_id; + sa.instance_simulate_migration_source(id, body.into_inner()).await?; Ok(HttpResponseUpdatedNoContent()) } diff --git a/sled-agent/src/sim/instance.rs b/sled-agent/src/sim/instance.rs index 33bc1c40c1..eb7ea0ca79 100644 --- a/sled-agent/src/sim/instance.rs +++ b/sled-agent/src/sim/instance.rs @@ -14,13 +14,14 @@ use nexus_client; use omicron_common::api::external::Error; use omicron_common::api::external::Generation; use omicron_common::api::external::ResourceType; -use omicron_common::api::internal::nexus::{SledInstanceState, VmmState}; +use omicron_common::api::internal::nexus::{SledVmmState, VmmState}; +use omicron_uuid_kinds::{GenericUuid, PropolisUuid}; use propolis_client::types::{ InstanceMigrateStatusResponse as PropolisMigrateResponse, InstanceMigrationStatus as PropolisMigrationStatus, InstanceState as PropolisInstanceState, InstanceStateMonitorResponse, }; -use sled_agent_types::instance::InstanceStateRequested; +use sled_agent_types::instance::VmmStateRequested; use std::collections::VecDeque; use std::sync::Arc; use std::sync::Mutex; @@ -170,13 +171,13 @@ impl SimInstanceInner { /// returning an action for the caller to simulate. fn request_transition( &mut self, - target: &InstanceStateRequested, + target: &VmmStateRequested, ) -> Result, Error> { match target { // When Nexus intends to migrate into a VMM, it should create that // VMM in the Migrating state and shouldn't request anything else // from it before asking to migrate in. - InstanceStateRequested::MigrationTarget(_) => { + VmmStateRequested::MigrationTarget(_) => { if !self.queue.is_empty() { return Err(Error::invalid_request(&format!( "can't request migration in with a non-empty state @@ -207,7 +208,7 @@ impl SimInstanceInner { SimulatedMigrationResult::Success, ); } - InstanceStateRequested::Running => { + VmmStateRequested::Running => { match self.next_resting_state() { VmmState::Starting => { self.queue_propolis_state( @@ -234,7 +235,7 @@ impl SimInstanceInner { } } } - InstanceStateRequested::Stopped => { + VmmStateRequested::Stopped => { match self.next_resting_state() { VmmState::Starting => { let mark_failed = false; @@ -256,7 +257,7 @@ impl SimInstanceInner { } } } - InstanceStateRequested::Reboot => match self.next_resting_state() { + VmmStateRequested::Reboot => match self.next_resting_state() { VmmState::Running => { // Further requests to reboot are ignored if the instance // is currently rebooting or about to reboot. @@ -315,7 +316,7 @@ impl SimInstanceInner { /// If the state change queue contains at least once instance state change, /// returns the requested instance state associated with the last instance /// state on the queue. Returns None otherwise. - fn desired(&self) -> Option { + fn desired(&self) -> Option { self.last_queued_instance_state().map(|terminal| match terminal { // State change requests may queue these states as intermediate // states, but the simulation (and the tests that rely on it) is @@ -331,13 +332,11 @@ impl SimInstanceInner { "pending resting state {:?} doesn't map to a requested state", terminal ), - PropolisInstanceState::Running => InstanceStateRequested::Running, + PropolisInstanceState::Running => VmmStateRequested::Running, PropolisInstanceState::Stopping | PropolisInstanceState::Stopped - | PropolisInstanceState::Destroyed => { - InstanceStateRequested::Stopped - } - PropolisInstanceState::Rebooting => InstanceStateRequested::Reboot, + | PropolisInstanceState::Destroyed => VmmStateRequested::Stopped, + PropolisInstanceState::Rebooting => VmmStateRequested::Reboot, }) } @@ -388,7 +387,7 @@ impl SimInstanceInner { /// Simulates rude termination by moving the instance to the Destroyed state /// immediately and clearing the queue of pending state transitions. - fn terminate(&mut self) -> SledInstanceState { + fn terminate(&mut self) -> SledVmmState { let mark_failed = false; self.state.terminate_rudely(mark_failed); self.queue.clear(); @@ -418,7 +417,7 @@ pub struct SimInstance { } impl SimInstance { - pub fn terminate(&self) -> SledInstanceState { + pub fn terminate(&self) -> SledVmmState { self.inner.lock().unwrap().terminate() } @@ -435,12 +434,12 @@ impl SimInstance { #[async_trait] impl Simulatable for SimInstance { - type CurrentState = SledInstanceState; - type RequestedState = InstanceStateRequested; + type CurrentState = SledVmmState; + type RequestedState = VmmStateRequested; type ProducerArgs = (); type Action = InstanceAction; - fn new(current: SledInstanceState) -> Self { + fn new(current: SledVmmState) -> Self { assert!(matches!( current.vmm_state.state, VmmState::Starting | VmmState::Migrating), @@ -453,7 +452,6 @@ impl Simulatable for SimInstance { inner: Arc::new(Mutex::new(SimInstanceInner { state: InstanceStates::new( current.vmm_state, - current.propolis_id, current.migration_in.map(|m| m.migration_id), ), last_response: InstanceStateMonitorResponse { @@ -480,7 +478,7 @@ impl Simulatable for SimInstance { fn request_transition( &mut self, - target: &InstanceStateRequested, + target: &VmmStateRequested, ) -> Result, Error> { self.inner.lock().unwrap().request_transition(target) } @@ -512,8 +510,8 @@ impl Simulatable for SimInstance { ) -> Result<(), Error> { nexus_client .cpapi_instances_put( - id, - &nexus_client::types::SledInstanceState::from(current), + &PropolisUuid::from_untyped_uuid(*id), + &nexus_client::types::SledVmmState::from(current), ) .await .map(|_| ()) diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 10536c8c80..aaac7f63d0 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -24,7 +24,7 @@ use omicron_common::api::external::{ ByteCount, DiskState, Error, Generation, ResourceType, }; use omicron_common::api::internal::nexus::{ - DiskRuntimeState, MigrationRuntimeState, MigrationState, SledInstanceState, + DiskRuntimeState, MigrationRuntimeState, MigrationState, SledVmmState, }; use omicron_common::api::internal::nexus::{ InstanceRuntimeState, VmmRuntimeState, @@ -35,8 +35,8 @@ use omicron_common::api::internal::shared::{ VirtualNetworkInterfaceHost, }; use omicron_common::disk::{ - DiskIdentity, DiskVariant, DisksManagementResult, - OmicronPhysicalDisksConfig, + DatasetsConfig, DatasetsManagementResult, DiskIdentity, DiskVariant, + DisksManagementResult, OmicronPhysicalDisksConfig, }; use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid, ZpoolUuid}; use oxnet::Ipv6Net; @@ -50,8 +50,7 @@ use sled_agent_types::early_networking::{ }; use sled_agent_types::instance::{ InstanceExternalIpBody, InstanceHardware, InstanceMetadata, - InstancePutStateResponse, InstanceStateRequested, - InstanceUnregisterResponse, + VmmPutStateResponse, VmmStateRequested, VmmUnregisterResponse, }; use slog::Logger; use std::collections::{HashMap, HashSet, VecDeque}; @@ -71,8 +70,8 @@ use uuid::Uuid; pub struct SledAgent { pub id: Uuid, pub ip: IpAddr, - /// collection of simulated instances, indexed by instance uuid - instances: Arc>, + /// collection of simulated VMMs, indexed by Propolis uuid + vmms: Arc>, /// collection of simulated disks, indexed by disk uuid disks: Arc>, storage: Mutex, @@ -84,7 +83,8 @@ pub struct SledAgent { mock_propolis: Mutex>, PropolisClient)>>, /// lists of external IPs assigned to instances - pub external_ips: Mutex>>, + pub external_ips: + Mutex>>, pub vpc_routes: Mutex>, config: Config, fake_zones: Mutex, @@ -170,7 +170,7 @@ impl SledAgent { Arc::new(SledAgent { id, ip: config.dropshot.bind_address.ip(), - instances: Arc::new(SimCollection::new( + vmms: Arc::new(SimCollection::new( Arc::clone(&nexus_client), instance_log, sim_mode, @@ -269,7 +269,7 @@ impl SledAgent { instance_runtime: InstanceRuntimeState, vmm_runtime: VmmRuntimeState, metadata: InstanceMetadata, - ) -> Result { + ) -> Result { // respond with a fake 500 level failure if asked to ensure an instance // with more than 16 CPUs. let ncpus: i64 = (&hardware.properties.ncpus).into(); @@ -317,11 +317,7 @@ impl SledAgent { // point to the correct address. let mock_lock = self.mock_propolis.lock().await; if let Some((_srv, client)) = mock_lock.as_ref() { - if !self - .instances - .contains_key(&instance_id.into_untyped_uuid()) - .await - { + if !self.vmms.contains_key(&instance_id.into_untyped_uuid()).await { let metadata = propolis_client::types::InstanceMetadata { project_id: metadata.project_id, silo_id: metadata.silo_id, @@ -379,12 +375,11 @@ impl SledAgent { }); let instance_run_time_state = self - .instances + .vmms .sim_ensure( - &instance_id.into_untyped_uuid(), - SledInstanceState { + &propolis_id.into_untyped_uuid(), + SledVmmState { vmm_state: vmm_runtime, - propolis_id, migration_in, migration_out: None, }, @@ -417,56 +412,53 @@ impl SledAgent { /// not notified. pub async fn instance_unregister( self: &Arc, - instance_id: InstanceUuid, - ) -> Result { + propolis_id: PropolisUuid, + ) -> Result { let instance = match self - .instances - .sim_get_cloned_object(&instance_id.into_untyped_uuid()) + .vmms + .sim_get_cloned_object(&propolis_id.into_untyped_uuid()) .await { Ok(instance) => instance, Err(Error::ObjectNotFound { .. }) => { - return Ok(InstanceUnregisterResponse { updated_runtime: None }) + return Ok(VmmUnregisterResponse { updated_runtime: None }) } Err(e) => return Err(e), }; - self.detach_disks_from_instance(instance_id).await?; - let response = InstanceUnregisterResponse { + let response = VmmUnregisterResponse { updated_runtime: Some(instance.terminate()), }; - self.instances.sim_force_remove(instance_id.into_untyped_uuid()).await; + self.vmms.sim_force_remove(propolis_id.into_untyped_uuid()).await; Ok(response) } /// Asks the supplied instance to transition to the requested state. pub async fn instance_ensure_state( self: &Arc, - instance_id: InstanceUuid, - state: InstanceStateRequested, - ) -> Result { + propolis_id: PropolisUuid, + state: VmmStateRequested, + ) -> Result { if let Some(e) = self.instance_ensure_state_error.lock().await.as_ref() { return Err(e.clone()); } let current = match self - .instances - .sim_get_cloned_object(&instance_id.into_untyped_uuid()) + .vmms + .sim_get_cloned_object(&propolis_id.into_untyped_uuid()) .await { Ok(i) => i.current().clone(), Err(_) => match state { - InstanceStateRequested::Stopped => { - return Ok(InstancePutStateResponse { - updated_runtime: None, - }); + VmmStateRequested::Stopped => { + return Ok(VmmPutStateResponse { updated_runtime: None }); } _ => { return Err(Error::invalid_request(&format!( - "instance {} not registered on sled", - instance_id, + "Propolis {} not registered on sled", + propolis_id, ))); } }, @@ -475,43 +467,41 @@ impl SledAgent { let mock_lock = self.mock_propolis.lock().await; if let Some((_srv, client)) = mock_lock.as_ref() { let body = match state { - InstanceStateRequested::MigrationTarget(_) => { + VmmStateRequested::MigrationTarget(_) => { return Err(Error::internal_error( "migration not implemented for mock Propolis", )); } - InstanceStateRequested::Running => { - let instances = self.instances.clone(); + VmmStateRequested::Running => { + let vmms = self.vmms.clone(); let log = self.log.new( o!("component" => "SledAgent-insure_instance_state"), ); tokio::spawn(async move { tokio::time::sleep(Duration::from_secs(10)).await; - match instances + match vmms .sim_ensure( - &instance_id.into_untyped_uuid(), + &propolis_id.into_untyped_uuid(), current, Some(state), ) .await { Ok(state) => { - let instance_state: nexus_client::types::SledInstanceState = state.into(); - info!(log, "sim_ensure success"; "instance_state" => #?instance_state); + let vmm_state: nexus_client::types::SledVmmState = state.into(); + info!(log, "sim_ensure success"; "vmm_state" => #?vmm_state); } Err(instance_put_error) => { error!(log, "sim_ensure failure"; "error" => #?instance_put_error); } } }); - return Ok(InstancePutStateResponse { - updated_runtime: None, - }); + return Ok(VmmPutStateResponse { updated_runtime: None }); } - InstanceStateRequested::Stopped => { + VmmStateRequested::Stopped => { propolis_client::types::InstanceStateRequested::Stop } - InstanceStateRequested::Reboot => { + VmmStateRequested::Reboot => { propolis_client::types::InstanceStateRequested::Reboot } }; @@ -521,30 +511,24 @@ impl SledAgent { } let new_state = self - .instances - .sim_ensure(&instance_id.into_untyped_uuid(), current, Some(state)) + .vmms + .sim_ensure(&propolis_id.into_untyped_uuid(), current, Some(state)) .await?; - // If this request will shut down the simulated instance, look for any - // disks that are attached to it and drive them to the Detached state. - if matches!(state, InstanceStateRequested::Stopped) { - self.detach_disks_from_instance(instance_id).await?; - } - - Ok(InstancePutStateResponse { updated_runtime: Some(new_state) }) + Ok(VmmPutStateResponse { updated_runtime: Some(new_state) }) } pub async fn instance_get_state( &self, - instance_id: InstanceUuid, - ) -> Result { + propolis_id: PropolisUuid, + ) -> Result { let instance = self - .instances - .sim_get_cloned_object(&instance_id.into_untyped_uuid()) + .vmms + .sim_get_cloned_object(&propolis_id.into_untyped_uuid()) .await .map_err(|_| { crate::sled_agent::Error::Instance( - crate::instance_manager::Error::NoSuchInstance(instance_id), + crate::instance_manager::Error::NoSuchVmm(propolis_id), ) })?; Ok(instance.current()) @@ -552,16 +536,16 @@ impl SledAgent { pub async fn instance_simulate_migration_source( &self, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, migration: instance::SimulateMigrationSource, ) -> Result<(), HttpError> { let instance = self - .instances - .sim_get_cloned_object(&instance_id.into_untyped_uuid()) + .vmms + .sim_get_cloned_object(&propolis_id.into_untyped_uuid()) .await .map_err(|_| { crate::sled_agent::Error::Instance( - crate::instance_manager::Error::NoSuchInstance(instance_id), + crate::instance_manager::Error::NoSuchVmm(propolis_id), ) })?; instance.set_simulated_migration_source(migration); @@ -572,25 +556,6 @@ impl SledAgent { *self.instance_ensure_state_error.lock().await = error; } - async fn detach_disks_from_instance( - &self, - instance_id: InstanceUuid, - ) -> Result<(), Error> { - self.disks - .sim_ensure_for_each_where( - |disk| match disk.current().disk_state { - DiskState::Attached(id) | DiskState::Attaching(id) => { - id == instance_id.into_untyped_uuid() - } - _ => false, - }, - &DiskStateRequested::Detached, - ) - .await?; - - Ok(()) - } - /// Idempotently ensures that the given API Disk (described by `api_disk`) /// is attached (or not) as specified. This simulates disk attach and /// detach, similar to instance boot and halt. @@ -607,16 +572,16 @@ impl SledAgent { &self.updates } - pub async fn instance_count(&self) -> usize { - self.instances.size().await + pub async fn vmm_count(&self) -> usize { + self.vmms.size().await } pub async fn disk_count(&self) -> usize { self.disks.size().await } - pub async fn instance_poke(&self, id: InstanceUuid, mode: PokeMode) { - self.instances.sim_poke(id.into_untyped_uuid(), mode).await; + pub async fn vmm_poke(&self, id: PropolisUuid, mode: PokeMode) { + self.vmms.sim_poke(id.into_untyped_uuid(), mode).await; } pub async fn disk_poke(&self, id: Uuid) { @@ -699,7 +664,7 @@ impl SledAgent { /// snapshot here. pub async fn instance_issue_disk_snapshot_request( &self, - _instance_id: InstanceUuid, + _propolis_id: PropolisUuid, disk_id: Uuid, snapshot_id: Uuid, ) -> Result<(), Error> { @@ -760,18 +725,17 @@ impl SledAgent { pub async fn instance_put_external_ip( &self, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, body_args: &InstanceExternalIpBody, ) -> Result<(), Error> { - if !self.instances.contains_key(&instance_id.into_untyped_uuid()).await - { + if !self.vmms.contains_key(&propolis_id.into_untyped_uuid()).await { return Err(Error::internal_error( - "can't alter IP state for nonexistent instance", + "can't alter IP state for VMM that's not registered", )); } let mut eips = self.external_ips.lock().await; - let my_eips = eips.entry(instance_id.into_untyped_uuid()).or_default(); + let my_eips = eips.entry(propolis_id).or_default(); // High-level behaviour: this should always succeed UNLESS // trying to add a double ephemeral. @@ -794,18 +758,17 @@ impl SledAgent { pub async fn instance_delete_external_ip( &self, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, body_args: &InstanceExternalIpBody, ) -> Result<(), Error> { - if !self.instances.contains_key(&instance_id.into_untyped_uuid()).await - { + if !self.vmms.contains_key(&propolis_id.into_untyped_uuid()).await { return Err(Error::internal_error( - "can't alter IP state for nonexistent instance", + "can't alter IP state for VMM that's not registered", )); } let mut eips = self.external_ips.lock().await; - let my_eips = eips.entry(instance_id.into_untyped_uuid()).or_default(); + let my_eips = eips.entry(propolis_id).or_default(); my_eips.remove(&body_args); @@ -905,6 +868,19 @@ impl SledAgent { }) } + pub async fn datasets_ensure( + &self, + config: DatasetsConfig, + ) -> Result { + self.storage.lock().await.datasets_ensure(config).await + } + + pub async fn datasets_config_list( + &self, + ) -> Result { + self.storage.lock().await.datasets_config_list().await + } + pub async fn omicron_physical_disks_list( &self, ) -> Result { diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 556388ce93..144fb48aa9 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -18,14 +18,17 @@ use crucible_agent_client::types::{ use dropshot::HandlerTaskMode; use dropshot::HttpError; use futures::lock::Mutex; +use omicron_common::disk::DatasetManagementStatus; +use omicron_common::disk::DatasetsConfig; +use omicron_common::disk::DatasetsManagementResult; use omicron_common::disk::DiskIdentity; use omicron_common::disk::DiskManagementStatus; use omicron_common::disk::DiskVariant; use omicron_common::disk::DisksManagementResult; use omicron_common::disk::OmicronPhysicalDisksConfig; use omicron_uuid_kinds::GenericUuid; -use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::PropolisUuid; use omicron_uuid_kinds::ZpoolUuid; use propolis_client::types::VolumeConstructionRequest; use slog::Logger; @@ -555,6 +558,7 @@ pub struct Storage { sled_id: Uuid, log: Logger, config: Option, + dataset_config: Option, physical_disks: HashMap, next_disk_slot: i64, zpools: HashMap, @@ -568,6 +572,7 @@ impl Storage { sled_id, log, config: None, + dataset_config: None, physical_disks: HashMap::new(), next_disk_slot: 0, zpools: HashMap::new(), @@ -581,6 +586,45 @@ impl Storage { &self.physical_disks } + pub async fn datasets_config_list( + &self, + ) -> Result { + let Some(config) = self.dataset_config.as_ref() else { + return Err(HttpError::for_not_found( + None, + "No control plane datasets".into(), + )); + }; + Ok(config.clone()) + } + + pub async fn datasets_ensure( + &mut self, + config: DatasetsConfig, + ) -> Result { + if let Some(stored_config) = self.dataset_config.as_ref() { + if stored_config.generation < config.generation { + return Err(HttpError::for_client_error( + None, + http::StatusCode::BAD_REQUEST, + "Generation number too old".to_string(), + )); + } + } + self.dataset_config.replace(config.clone()); + + Ok(DatasetsManagementResult { + status: config + .datasets + .values() + .map(|config| DatasetManagementStatus { + dataset_name: config.name.clone(), + err: None, + }) + .collect(), + }) + } + pub async fn omicron_physical_disks_list( &mut self, ) -> Result { @@ -869,7 +913,7 @@ impl Pantry { self.sled_agent .instance_issue_disk_snapshot_request( - InstanceUuid::new_v4(), // instance id, not used by function + PropolisUuid::new_v4(), // instance id, not used by function volume_id.parse().unwrap(), snapshot_id.parse().unwrap(), ) diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 50e5611027..f13d8caccf 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -38,9 +38,7 @@ use omicron_common::address::{ get_sled_address, get_switch_zone_address, Ipv6Subnet, SLED_PREFIX, }; use omicron_common::api::external::{ByteCount, ByteCountRangeError, Vni}; -use omicron_common::api::internal::nexus::{ - SledInstanceState, VmmRuntimeState, -}; +use omicron_common::api::internal::nexus::{SledVmmState, VmmRuntimeState}; use omicron_common::api::internal::shared::{ HostPortConfig, RackNetworkConfig, ResolvedVpcFirewallRule, ResolvedVpcRouteSet, ResolvedVpcRouteState, SledIdentifiers, @@ -53,7 +51,10 @@ use omicron_common::api::{ use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, BackoffError, }; -use omicron_common::disk::{DisksManagementResult, OmicronPhysicalDisksConfig}; +use omicron_common::disk::{ + DatasetsConfig, DatasetsManagementResult, DisksManagementResult, + OmicronPhysicalDisksConfig, +}; use omicron_ddm_admin_client::Client as DdmAdminClient; use omicron_uuid_kinds::{InstanceUuid, PropolisUuid}; use sled_agent_api::Zpool; @@ -61,8 +62,7 @@ use sled_agent_types::disk::DiskStateRequested; use sled_agent_types::early_networking::EarlyNetworkConfig; use sled_agent_types::instance::{ InstanceExternalIpBody, InstanceHardware, InstanceMetadata, - InstancePutStateResponse, InstanceStateRequested, - InstanceUnregisterResponse, + VmmPutStateResponse, VmmStateRequested, VmmUnregisterResponse, }; use sled_agent_types::sled::{BaseboardId, StartSledAgentRequest}; use sled_agent_types::time_sync::TimeSync; @@ -227,7 +227,7 @@ impl From for dropshot::HttpError { } } Error::Instance( - e @ crate::instance_manager::Error::NoSuchInstance(_), + e @ crate::instance_manager::Error::NoSuchVmm(_), ) => HttpError::for_not_found( Some(NO_SUCH_INSTANCE.to_string()), e.to_string(), @@ -811,6 +811,29 @@ impl SledAgent { self.inner.zone_bundler.cleanup().await.map_err(Error::from) } + pub async fn datasets_config_list(&self) -> Result { + Ok(self.storage().datasets_config_list().await?) + } + + pub async fn datasets_ensure( + &self, + config: DatasetsConfig, + ) -> Result { + info!(self.log, "datasets ensure"); + let datasets_result = self.storage().datasets_ensure(config).await?; + info!(self.log, "datasets ensure: Updated storage"); + + // TODO(https://github.com/oxidecomputer/omicron/issues/6177): + // At the moment, we don't actually remove any datasets -- this function + // just adds new datasets. + // + // Once we start removing old datasets, we should probably ensure that + // they are not longer in-use before returning (similar to + // omicron_physical_disks_ensure). + + Ok(datasets_result) + } + /// Requests the set of physical disks currently managed by the Sled Agent. /// /// This should be contrasted by the set of disks in the inventory, which @@ -899,7 +922,7 @@ impl SledAgent { &self, requested_zones: OmicronZonesConfig, ) -> Result<(), Error> { - // TODO: + // TODO(https://github.com/oxidecomputer/omicron/issues/6043): // - If these are the set of filesystems, we should also consider // removing the ones which are not listed here. // - It's probably worth sending a bulk request to the storage system, @@ -966,7 +989,7 @@ impl SledAgent { vmm_runtime: VmmRuntimeState, propolis_addr: SocketAddr, metadata: InstanceMetadata, - ) -> Result { + ) -> Result { self.inner .instances .ensure_registered( @@ -990,11 +1013,11 @@ impl SledAgent { /// rudely terminates the instance. pub async fn instance_ensure_unregistered( &self, - instance_id: InstanceUuid, - ) -> Result { + propolis_id: PropolisUuid, + ) -> Result { self.inner .instances - .ensure_unregistered(instance_id) + .ensure_unregistered(propolis_id) .await .map_err(|e| Error::Instance(e)) } @@ -1003,12 +1026,12 @@ impl SledAgent { /// state. pub async fn instance_ensure_state( &self, - instance_id: InstanceUuid, - target: InstanceStateRequested, - ) -> Result { + propolis_id: PropolisUuid, + target: VmmStateRequested, + ) -> Result { self.inner .instances - .ensure_state(instance_id, target) + .ensure_state(propolis_id, target) .await .map_err(|e| Error::Instance(e)) } @@ -1020,12 +1043,12 @@ impl SledAgent { /// does not match the current ephemeral IP. pub async fn instance_put_external_ip( &self, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, external_ip: &InstanceExternalIpBody, ) -> Result<(), Error> { self.inner .instances - .add_external_ip(instance_id, external_ip) + .add_external_ip(propolis_id, external_ip) .await .map_err(|e| Error::Instance(e)) } @@ -1034,12 +1057,12 @@ impl SledAgent { /// specified external IP address in either its ephemeral or floating IP set. pub async fn instance_delete_external_ip( &self, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, external_ip: &InstanceExternalIpBody, ) -> Result<(), Error> { self.inner .instances - .delete_external_ip(instance_id, external_ip) + .delete_external_ip(propolis_id, external_ip) .await .map_err(|e| Error::Instance(e)) } @@ -1047,11 +1070,11 @@ impl SledAgent { /// Returns the state of the instance with the provided ID. pub async fn instance_get_state( &self, - instance_id: InstanceUuid, - ) -> Result { + propolis_id: PropolisUuid, + ) -> Result { self.inner .instances - .get_instance_state(instance_id) + .get_instance_state(propolis_id) .await .map_err(|e| Error::Instance(e)) } @@ -1082,19 +1105,15 @@ impl SledAgent { } /// Issue a snapshot request for a Crucible disk attached to an instance - pub async fn instance_issue_disk_snapshot_request( + pub async fn vmm_issue_disk_snapshot_request( &self, - instance_id: InstanceUuid, + propolis_id: PropolisUuid, disk_id: Uuid, snapshot_id: Uuid, ) -> Result<(), Error> { self.inner .instances - .instance_issue_disk_snapshot_request( - instance_id, - disk_id, - snapshot_id, - ) + .issue_disk_snapshot_request(propolis_id, disk_id, snapshot_id) .await .map_err(Error::from) } diff --git a/sled-agent/types/src/instance.rs b/sled-agent/types/src/instance.rs index 0753e273dc..a39fae414b 100644 --- a/sled-agent/types/src/instance.rs +++ b/sled-agent/types/src/instance.rs @@ -11,14 +11,14 @@ use std::{ use omicron_common::api::internal::{ nexus::{ - InstanceProperties, InstanceRuntimeState, SledInstanceState, - VmmRuntimeState, + InstanceProperties, InstanceRuntimeState, SledVmmState, VmmRuntimeState, }, shared::{ DhcpConfig, NetworkInterface, ResolvedVpcFirewallRule, SourceNatConfig, }, }; -use omicron_uuid_kinds::PropolisUuid; +use omicron_common::NoDebug; +use omicron_uuid_kinds::InstanceUuid; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -37,10 +37,8 @@ pub struct InstanceEnsureBody { /// The initial VMM runtime state for the VMM being registered. pub vmm_runtime: VmmRuntimeState, - /// The ID of the VMM being registered. This may not be the active VMM ID in - /// the instance runtime state (e.g. if the new VMM is going to be a - /// migration target). - pub propolis_id: PropolisUuid, + /// The ID of the instance for which this VMM is being created. + pub instance_id: InstanceUuid, /// The address at which this VMM should serve a Propolis server API. pub propolis_addr: SocketAddr, @@ -63,7 +61,7 @@ pub struct InstanceHardware { pub dhcp_config: DhcpConfig, // TODO: replace `propolis_client::*` with locally-modeled request type pub disks: Vec, - pub cloud_init_bytes: Option, + pub cloud_init_bytes: Option>, } /// Metadata used to track statistics about an instance. @@ -80,19 +78,19 @@ pub struct InstanceMetadata { /// The body of a request to move a previously-ensured instance into a specific /// runtime state. #[derive(Serialize, Deserialize, JsonSchema)] -pub struct InstancePutStateBody { +pub struct VmmPutStateBody { /// The state into which the instance should be driven. - pub state: InstanceStateRequested, + pub state: VmmStateRequested, } /// The response sent from a request to move an instance into a specific runtime /// state. #[derive(Debug, Serialize, Deserialize, JsonSchema)] -pub struct InstancePutStateResponse { +pub struct VmmPutStateResponse { /// The current runtime state of the instance after handling the request to /// change its state. If the instance's state did not change, this field is /// `None`. - pub updated_runtime: Option, + pub updated_runtime: Option, } /// Requestable running state of an Instance. @@ -100,7 +98,7 @@ pub struct InstancePutStateResponse { /// A subset of [`omicron_common::api::external::InstanceState`]. #[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "snake_case", tag = "type", content = "value")] -pub enum InstanceStateRequested { +pub enum VmmStateRequested { /// Run this instance by migrating in from a previous running incarnation of /// the instance. MigrationTarget(InstanceMigrationTargetParams), @@ -113,40 +111,40 @@ pub enum InstanceStateRequested { Reboot, } -impl fmt::Display for InstanceStateRequested { +impl fmt::Display for VmmStateRequested { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.label()) } } -impl InstanceStateRequested { +impl VmmStateRequested { fn label(&self) -> &str { match self { - InstanceStateRequested::MigrationTarget(_) => "migrating in", - InstanceStateRequested::Running => "running", - InstanceStateRequested::Stopped => "stopped", - InstanceStateRequested::Reboot => "reboot", + VmmStateRequested::MigrationTarget(_) => "migrating in", + VmmStateRequested::Running => "running", + VmmStateRequested::Stopped => "stopped", + VmmStateRequested::Reboot => "reboot", } } /// Returns true if the state represents a stopped Instance. pub fn is_stopped(&self) -> bool { match self { - InstanceStateRequested::MigrationTarget(_) => false, - InstanceStateRequested::Running => false, - InstanceStateRequested::Stopped => true, - InstanceStateRequested::Reboot => false, + VmmStateRequested::MigrationTarget(_) => false, + VmmStateRequested::Running => false, + VmmStateRequested::Stopped => true, + VmmStateRequested::Reboot => false, } } } /// The response sent from a request to unregister an instance. #[derive(Serialize, Deserialize, JsonSchema)] -pub struct InstanceUnregisterResponse { +pub struct VmmUnregisterResponse { /// The current state of the instance after handling the request to /// unregister it. If the instance's state did not change, this field is /// `None`. - pub updated_runtime: Option, + pub updated_runtime: Option, } /// Parameters used when directing Propolis to initialize itself via live diff --git a/sled-storage/src/dataset.rs b/sled-storage/src/dataset.rs index 74f2be782f..e2b024db11 100644 --- a/sled-storage/src/dataset.rs +++ b/sled-storage/src/dataset.rs @@ -15,10 +15,10 @@ use illumos_utils::zfs::{ use illumos_utils::zpool::ZpoolName; use key_manager::StorageKeyRequester; use omicron_common::api::internal::shared::DatasetKind; -use omicron_common::disk::{DiskIdentity, DiskVariant}; +use omicron_common::disk::{ + CompressionAlgorithm, DatasetName, DiskIdentity, DiskVariant, GzipLevel, +}; use rand::distributions::{Alphanumeric, DistString}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; use slog::{debug, info, Logger}; use std::process::Stdio; use std::str::FromStr; @@ -45,7 +45,8 @@ cfg_if! { // tuned as needed. pub const DUMP_DATASET_QUOTA: usize = 100 * (1 << 30); // passed to zfs create -o compression= -pub const DUMP_DATASET_COMPRESSION: &'static str = "gzip-9"; +pub const DUMP_DATASET_COMPRESSION: CompressionAlgorithm = + CompressionAlgorithm::GzipN { level: GzipLevel::new::<9>() }; // U.2 datasets live under the encrypted dataset and inherit encryption pub const ZONE_DATASET: &'static str = "crypt/zone"; @@ -102,12 +103,17 @@ struct ExpectedDataset { // Identifies if the dataset should be deleted on boot wipe: bool, // Optional compression mode - compression: Option<&'static str>, + compression: CompressionAlgorithm, } impl ExpectedDataset { const fn new(name: &'static str) -> Self { - ExpectedDataset { name, quota: None, wipe: false, compression: None } + ExpectedDataset { + name, + quota: None, + wipe: false, + compression: CompressionAlgorithm::Off, + } } const fn quota(mut self, quota: usize) -> Self { @@ -120,151 +126,12 @@ impl ExpectedDataset { self } - const fn compression(mut self, compression: &'static str) -> Self { - self.compression = Some(compression); + const fn compression(mut self, compression: CompressionAlgorithm) -> Self { + self.compression = compression; self } } -/// The type of a dataset, and an auxiliary information necessary to -/// successfully launch a zone managing the associated data. -/// -/// There is currently no auxiliary data here, but there's a separation from -/// omicron-common's `DatasetKind` in case there might be some in the future. -#[derive( - Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, -)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum DatasetType { - // TODO: `DatasetKind` uses `Cockroach`, not `CockroachDb`, for historical - // reasons. It may be worth using the same name for both. - CockroachDb, - Crucible, - Clickhouse, - ClickhouseKeeper, - ClickhouseServer, - ExternalDns, - InternalDns, -} - -impl DatasetType { - pub fn dataset_should_be_encrypted(&self) -> bool { - match self { - // We encrypt all datasets except Crucible. - // - // Crucible already performs encryption internally, and we - // avoid double-encryption. - DatasetType::Crucible => false, - _ => true, - } - } - - pub fn kind(&self) -> DatasetKind { - match self { - Self::Crucible => DatasetKind::Crucible, - Self::CockroachDb => DatasetKind::Cockroach, - Self::Clickhouse => DatasetKind::Clickhouse, - Self::ClickhouseKeeper => DatasetKind::ClickhouseKeeper, - Self::ClickhouseServer => DatasetKind::ClickhouseServer, - Self::ExternalDns => DatasetKind::ExternalDns, - Self::InternalDns => DatasetKind::InternalDns, - } - } -} - -#[derive(Debug, thiserror::Error)] -pub enum DatasetKindParseError { - #[error("Dataset unknown: {0}")] - UnknownDataset(String), -} - -impl FromStr for DatasetType { - type Err = DatasetKindParseError; - - fn from_str(s: &str) -> Result { - use DatasetType::*; - let kind = match s { - "crucible" => Crucible, - "cockroachdb" => CockroachDb, - "clickhouse" => Clickhouse, - "clickhouse_keeper" => ClickhouseKeeper, - "external_dns" => ExternalDns, - "internal_dns" => InternalDns, - _ => { - return Err(DatasetKindParseError::UnknownDataset( - s.to_string(), - )) - } - }; - Ok(kind) - } -} - -impl std::fmt::Display for DatasetType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use DatasetType::*; - let s = match self { - Crucible => "crucible", - CockroachDb => "cockroachdb", - Clickhouse => "clickhouse", - ClickhouseKeeper => "clickhouse_keeper", - ClickhouseServer => "clickhouse_server", - ExternalDns => "external_dns", - InternalDns => "internal_dns", - }; - write!(f, "{}", s) - } -} - -#[derive( - Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone, JsonSchema, -)] -pub struct DatasetName { - // A unique identifier for the Zpool on which the dataset is stored. - pool_name: ZpoolName, - // A name for the dataset within the Zpool. - kind: DatasetType, -} - -impl DatasetName { - pub fn new(pool_name: ZpoolName, kind: DatasetType) -> Self { - Self { pool_name, kind } - } - - pub fn pool(&self) -> &ZpoolName { - &self.pool_name - } - - pub fn dataset(&self) -> &DatasetType { - &self.kind - } - - /// Returns the full name of the dataset, as would be returned from - /// "zfs get" or "zfs list". - /// - /// If this dataset should be encrypted, this automatically adds the - /// "crypt" dataset component. - pub fn full_name(&self) -> String { - // Currently, we encrypt all datasets except Crucible. - // - // Crucible already performs encryption internally, and we - // avoid double-encryption. - if self.kind.dataset_should_be_encrypted() { - self.full_encrypted_name() - } else { - self.full_unencrypted_name() - } - } - - fn full_encrypted_name(&self) -> String { - format!("{}/crypt/{}", self.pool_name, self.kind) - } - - fn full_unencrypted_name(&self) -> String { - format!("{}/{}", self.pool_name, self.kind) - } -} - #[derive(Debug, thiserror::Error)] pub enum DatasetError { #[error("Cannot open {path} due to {error}")] @@ -431,6 +298,7 @@ pub(crate) async fn ensure_zpool_has_datasets( let encryption_details = None; let size_details = Some(SizeDetails { quota: dataset.quota, + reservation: None, compression: dataset.compression, }); Zfs::ensure_filesystem( @@ -577,7 +445,7 @@ async fn ensure_zpool_dataset_is_encrypted( zpool_name: &ZpoolName, unencrypted_dataset: &str, ) -> Result<(), DatasetEncryptionMigrationError> { - let Ok(kind) = DatasetType::from_str(&unencrypted_dataset) else { + let Ok(kind) = DatasetKind::from_str(&unencrypted_dataset) else { info!(log, "Unrecognized dataset kind"); return Ok(()); }; @@ -818,7 +686,7 @@ mod test { #[test] fn serialize_dataset_name() { let pool = ZpoolName::new_internal(ZpoolUuid::new_v4()); - let kind = DatasetType::Crucible; + let kind = DatasetKind::Crucible; let name = DatasetName::new(pool, kind); serde_json::to_string(&name).unwrap(); } diff --git a/sled-storage/src/error.rs b/sled-storage/src/error.rs index 4c5582fd79..988f7f363a 100644 --- a/sled-storage/src/error.rs +++ b/sled-storage/src/error.rs @@ -4,11 +4,12 @@ //! Storage related errors -use crate::dataset::{DatasetError, DatasetName}; +use crate::dataset::DatasetError; use crate::disk::DiskError; use camino::Utf8PathBuf; use omicron_common::api::external::ByteCountRangeError; use omicron_common::api::external::Generation; +use omicron_common::disk::DatasetName; use uuid::Uuid; #[derive(thiserror::Error, Debug)] @@ -83,6 +84,15 @@ pub enum Error { current: Generation, }, + #[error("Invalid configuration (UUID mismatch in arguments)")] + ConfigUuidMismatch, + + #[error("Dataset configuration out-of-date (asked for {requested}, but latest is {current})")] + DatasetConfigurationOutdated { requested: Generation, current: Generation }, + + #[error("Dataset configuration changed for the same generation number: {generation}")] + DatasetConfigurationChanged { generation: Generation }, + #[error("Failed to update ledger in internal storage")] Ledger(#[from] omicron_common::ledger::Error), diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs index 3cbf00530a..88e1bbaa34 100644 --- a/sled-storage/src/manager.rs +++ b/sled-storage/src/manager.rs @@ -7,7 +7,7 @@ use std::collections::HashSet; use crate::config::MountConfig; -use crate::dataset::{DatasetName, CONFIG_DATASET}; +use crate::dataset::CONFIG_DATASET; use crate::disk::RawDisk; use crate::error::Error; use crate::resources::{AllDisks, StorageResources}; @@ -18,11 +18,14 @@ use illumos_utils::zfs::{Mountpoint, Zfs}; use illumos_utils::zpool::ZpoolName; use key_manager::StorageKeyRequester; use omicron_common::disk::{ - DiskIdentity, DiskVariant, DisksManagementResult, + DatasetConfig, DatasetManagementStatus, DatasetName, DatasetsConfig, + DatasetsManagementResult, DiskIdentity, DiskVariant, DisksManagementResult, OmicronPhysicalDisksConfig, }; use omicron_common::ledger::Ledger; -use slog::{info, o, warn, Logger}; +use omicron_uuid_kinds::DatasetUuid; +use omicron_uuid_kinds::GenericUuid; +use slog::{error, info, o, warn, Logger}; use std::future::Future; use tokio::sync::{mpsc, oneshot, watch}; use tokio::time::{interval, Duration, MissedTickBehavior}; @@ -62,6 +65,9 @@ const SYNCHRONIZE_INTERVAL: Duration = Duration::from_secs(10); // The filename of the ledger storing physical disk info const DISKS_LEDGER_FILENAME: &str = "omicron-physical-disks.json"; +// The filename of the ledger storing dataset info +const DATASETS_LEDGER_FILENAME: &str = "omicron-datasets.json"; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum StorageManagerState { // We know that any attempts to manage disks will fail, as the key manager @@ -114,6 +120,16 @@ pub(crate) enum StorageRequest { tx: DebugIgnore>>, }, + DatasetsEnsure { + config: DatasetsConfig, + tx: DebugIgnore< + oneshot::Sender>, + >, + }, + DatasetsList { + tx: DebugIgnore>>, + }, + // Requests to explicitly manage or stop managing a set of devices OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig, @@ -240,6 +256,31 @@ impl StorageHandle { rx.map(|result| result.unwrap()) } + pub async fn datasets_ensure( + &self, + config: DatasetsConfig, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.tx + .send(StorageRequest::DatasetsEnsure { config, tx: tx.into() }) + .await + .unwrap(); + + rx.await.unwrap() + } + + /// Reads the last value written to storage by + /// [Self::datasets_ensure]. + pub async fn datasets_config_list(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.tx + .send(StorageRequest::DatasetsList { tx: tx.into() }) + .await + .unwrap(); + + rx.await.unwrap() + } + pub async fn omicron_physical_disks_ensure( &self, config: OmicronPhysicalDisksConfig, @@ -322,6 +363,10 @@ impl StorageHandle { rx.await.unwrap() } + // TODO(https://github.com/oxidecomputer/omicron/issues/6043): + // + // Deprecate usage of this function, prefer to call "datasets_ensure" + // and ask for the set of all datasets from Nexus. pub async fn upsert_filesystem( &self, dataset_id: Uuid, @@ -428,6 +473,12 @@ impl StorageManager { self.ensure_using_exactly_these_disks(raw_disks).await; let _ = tx.0.send(Ok(())); } + StorageRequest::DatasetsEnsure { config, tx } => { + let _ = tx.0.send(self.datasets_ensure(config).await); + } + StorageRequest::DatasetsList { tx } => { + let _ = tx.0.send(self.datasets_config_list().await); + } StorageRequest::OmicronPhysicalDisksEnsure { config, tx } => { let _ = tx.0.send(self.omicron_physical_disks_ensure(config).await); @@ -485,6 +536,10 @@ impl StorageManager { ); } + // Sled Agents can remember which disks they need to manage by reading + // a configuration file from the M.2s. + // + // This function returns the paths to those configuration files. async fn all_omicron_disk_ledgers(&self) -> Vec { self.resources .disks() @@ -494,6 +549,19 @@ impl StorageManager { .collect() } + // Sled Agents can remember which datasets they need to manage by reading + // a configuration file from the M.2s. + // + // This function returns the paths to those configuration files. + async fn all_omicron_dataset_ledgers(&self) -> Vec { + self.resources + .disks() + .all_m2_mountpoints(CONFIG_DATASET) + .into_iter() + .map(|p| p.join(DATASETS_LEDGER_FILENAME)) + .collect() + } + // Manages a newly detected disk that has been attached to this sled. // // For U.2s: we update our inventory. @@ -545,9 +613,11 @@ impl StorageManager { self.resources.insert_or_update_disk(raw_disk).await } - async fn load_ledger(&self) -> Option> { + async fn load_disks_ledger( + &self, + ) -> Option> { let ledger_paths = self.all_omicron_disk_ledgers().await; - let log = self.log.new(o!("request" => "load_ledger")); + let log = self.log.new(o!("request" => "load_disks_ledger")); let maybe_ledger = Ledger::::new( &log, ledger_paths.clone(), @@ -579,7 +649,7 @@ impl StorageManager { // Now that we're actually able to unpack U.2s, attempt to load the // set of disks which we previously stored in the ledger, if one // existed. - let ledger = self.load_ledger().await; + let ledger = self.load_disks_ledger().await; if let Some(ledger) = ledger { info!(self.log, "Setting StorageResources state to match ledger"); @@ -591,9 +661,160 @@ impl StorageManager { info!(self.log, "KeyManager ready, but no ledger detected"); } + // We don't load any configuration for datasets, since we aren't + // currently storing any dataset information in-memory. + // + // If we ever wanted to do so, however, we could load that information + // here. + Ok(()) } + async fn datasets_ensure( + &mut self, + config: DatasetsConfig, + ) -> Result { + let log = self.log.new(o!("request" => "datasets_ensure")); + + // As a small input-check, confirm that the UUID of the map of inputs + // matches the DatasetConfig. + // + // The dataset configs are sorted by UUID so they always appear in the + // same order, but this check prevents adding an entry of: + // - (UUID: X, Config(UUID: Y)), for X != Y + if !config.datasets.iter().all(|(id, config)| *id == config.id) { + return Err(Error::ConfigUuidMismatch); + } + + // We rely on the schema being stable across reboots -- observe + // "test_datasets_schema" below for that property guarantee. + let ledger_paths = self.all_omicron_dataset_ledgers().await; + let maybe_ledger = + Ledger::::new(&log, ledger_paths.clone()).await; + + let mut ledger = match maybe_ledger { + Some(ledger) => { + info!( + log, + "Comparing 'requested datasets' to ledger on internal storage" + ); + let ledger_data = ledger.data(); + if config.generation < ledger_data.generation { + warn!( + log, + "Request looks out-of-date compared to prior request"; + "requested_generation" => ?config.generation, + "ledger_generation" => ?ledger_data.generation, + ); + return Err(Error::DatasetConfigurationOutdated { + requested: config.generation, + current: ledger_data.generation, + }); + } else if config.generation == ledger_data.generation { + info!( + log, + "Requested generation number matches prior request", + ); + + if ledger_data != &config { + error!( + log, + "Requested configuration changed (with the same generation)"; + "generation" => ?config.generation + ); + return Err(Error::DatasetConfigurationChanged { + generation: config.generation, + }); + } + } else { + info!( + log, + "Request looks newer than prior requests"; + "requested_generation" => ?config.generation, + "ledger_generation" => ?ledger_data.generation, + ); + } + ledger + } + None => { + info!(log, "No previously-stored 'requested datasets', creating new ledger"); + Ledger::::new_with( + &log, + ledger_paths.clone(), + DatasetsConfig::default(), + ) + } + }; + + let result = self.datasets_ensure_internal(&log, &config).await; + + let ledger_data = ledger.data_mut(); + if *ledger_data == config { + return Ok(result); + } + *ledger_data = config; + ledger.commit().await?; + + Ok(result) + } + + // Attempts to ensure that each dataset exist. + // + // Does not return an error, because the [DatasetsManagementResult] type + // includes details about all possible errors that may occur on + // a per-dataset granularity. + async fn datasets_ensure_internal( + &mut self, + log: &Logger, + config: &DatasetsConfig, + ) -> DatasetsManagementResult { + let mut status = vec![]; + for dataset in config.datasets.values() { + status.push(self.dataset_ensure_internal(log, dataset).await); + } + DatasetsManagementResult { status } + } + + async fn dataset_ensure_internal( + &mut self, + log: &Logger, + config: &DatasetConfig, + ) -> DatasetManagementStatus { + let log = log.new(o!("name" => config.name.full_name())); + info!(log, "Ensuring dataset"); + let mut status = DatasetManagementStatus { + dataset_name: config.name.clone(), + err: None, + }; + + if let Err(err) = self.ensure_dataset(config).await { + warn!(log, "Failed to ensure dataset"; "dataset" => ?status.dataset_name, "err" => ?err); + status.err = Some(err.to_string()); + }; + + status + } + + // Lists datasets that this sled is configured to use. + async fn datasets_config_list(&mut self) -> Result { + let log = self.log.new(o!("request" => "datasets_config_list")); + + let ledger_paths = self.all_omicron_dataset_ledgers().await; + let maybe_ledger = + Ledger::::new(&log, ledger_paths.clone()).await; + + match maybe_ledger { + Some(ledger) => { + info!(log, "Found ledger on internal storage"); + return Ok(ledger.data().clone()); + } + None => { + info!(log, "No ledger detected on internal storage"); + return Err(Error::LedgerNotFound); + } + } + } + // Makes an U.2 disk managed by the control plane within [`StorageResources`]. async fn omicron_physical_disks_ensure( &mut self, @@ -765,6 +986,77 @@ impl StorageManager { } } + // Ensures a dataset exists within a zpool, according to `config`. + async fn ensure_dataset( + &mut self, + config: &DatasetConfig, + ) -> Result<(), Error> { + info!(self.log, "ensure_dataset"; "config" => ?config); + + // We can only place datasets within managed disks. + // If a disk is attached to this sled, but not a part of the Control + // Plane, it is treated as "not found" for dataset placement. + if !self + .resources + .disks() + .iter_managed() + .any(|(_, disk)| disk.zpool_name() == config.name.pool()) + { + return Err(Error::ZpoolNotFound(format!( + "{}", + config.name.pool(), + ))); + } + + let zoned = config.name.dataset().zoned(); + let mountpoint_path = if zoned { + Utf8PathBuf::from("/data") + } else { + config.name.pool().dataset_mountpoint( + &Utf8PathBuf::from("/"), + &config.name.dataset().to_string(), + ) + }; + let mountpoint = Mountpoint::Path(mountpoint_path); + + let fs_name = &config.name.full_name(); + let do_format = true; + + // The "crypt" dataset needs these details, but should already exist + // by the time we're creating datasets inside. + let encryption_details = None; + let size_details = Some(illumos_utils::zfs::SizeDetails { + quota: config.quota, + reservation: config.reservation, + compression: config.compression, + }); + Zfs::ensure_filesystem( + fs_name, + mountpoint, + zoned, + do_format, + encryption_details, + size_details, + None, + )?; + // Ensure the dataset has a usable UUID. + if let Ok(id_str) = Zfs::get_oxide_value(&fs_name, "uuid") { + if let Ok(id) = id_str.parse::() { + if id != config.id { + return Err(Error::UuidMismatch { + name: Box::new(config.name.clone()), + old: id.into_untyped_uuid(), + new: config.id.into_untyped_uuid(), + }); + } + return Ok(()); + } + } + Zfs::set_oxide_value(&fs_name, "uuid", &config.id.to_string())?; + + Ok(()) + } + // Attempts to add a dataset within a zpool, according to `request`. async fn add_dataset( &mut self, @@ -824,16 +1116,19 @@ impl StorageManager { /// systems. #[cfg(all(test, target_os = "illumos"))] mod tests { - use crate::dataset::DatasetType; use crate::disk::RawSyntheticDisk; use crate::manager_test_harness::StorageManagerTestHarness; use super::*; use camino_tempfile::tempdir_in; + use omicron_common::api::external::Generation; + use omicron_common::disk::CompressionAlgorithm; + use omicron_common::disk::DatasetKind; use omicron_common::disk::DiskManagementError; use omicron_common::ledger; use omicron_test_utils::dev::test_setup_log; use sled_hardware::DiskFirmware; + use std::collections::BTreeMap; use std::sync::atomic::Ordering; use uuid::Uuid; @@ -1299,7 +1594,7 @@ mod tests { let dataset_id = Uuid::new_v4(); let zpool_name = ZpoolName::new_external(config.disks[0].pool_id); let dataset_name = - DatasetName::new(zpool_name.clone(), DatasetType::Crucible); + DatasetName::new(zpool_name.clone(), DatasetKind::Crucible); harness .handle() .upsert_filesystem(dataset_id, dataset_name) @@ -1309,6 +1604,86 @@ mod tests { harness.cleanup().await; logctx.cleanup_successful(); } + + #[tokio::test] + async fn ensure_datasets() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log("ensure_datasets"); + let mut harness = StorageManagerTestHarness::new(&logctx.log).await; + + // Test setup: Add a U.2 and M.2, adopt them into the "control plane" + // for usage. + harness.handle().key_manager_ready().await; + let raw_disks = + harness.add_vdevs(&["u2_under_test.vdev", "m2_helping.vdev"]).await; + let config = harness.make_config(1, &raw_disks); + let result = harness + .handle() + .omicron_physical_disks_ensure(config.clone()) + .await + .expect("Ensuring disks should work after key manager is ready"); + assert!(!result.has_error(), "{:?}", result); + + // Create a dataset on the newly formatted U.2 + let id = DatasetUuid::new_v4(); + let zpool_name = ZpoolName::new_external(config.disks[0].pool_id); + let name = DatasetName::new(zpool_name.clone(), DatasetKind::Crucible); + let datasets = BTreeMap::from([( + id, + DatasetConfig { + id, + name, + compression: CompressionAlgorithm::Off, + quota: None, + reservation: None, + }, + )]); + // "Generation = 1" is reserved as "no requests seen yet", so we jump + // past it. + let generation = Generation::new().next(); + let mut config = DatasetsConfig { generation, datasets }; + + let status = + harness.handle().datasets_ensure(config.clone()).await.unwrap(); + assert!(!status.has_error()); + + // List datasets, expect to see what we just created + let observed_config = + harness.handle().datasets_config_list().await.unwrap(); + assert_eq!(config, observed_config); + + // Calling "datasets_ensure" with the same input should succeed. + let status = + harness.handle().datasets_ensure(config.clone()).await.unwrap(); + assert!(!status.has_error()); + + let current_config_generation = config.generation; + let next_config_generation = config.generation.next(); + + // Calling "datasets_ensure" with an old generation should fail + config.generation = Generation::new(); + let err = + harness.handle().datasets_ensure(config.clone()).await.unwrap_err(); + assert!(matches!(err, Error::DatasetConfigurationOutdated { .. })); + + // However, calling it with a different input and the same generation + // number should fail. + config.generation = current_config_generation; + config.datasets.values_mut().next().unwrap().reservation = Some(1024); + let err = + harness.handle().datasets_ensure(config.clone()).await.unwrap_err(); + assert!(matches!(err, Error::DatasetConfigurationChanged { .. })); + + // If we bump the generation number while making a change, updated + // configs will work. + config.generation = next_config_generation; + let status = + harness.handle().datasets_ensure(config.clone()).await.unwrap(); + assert!(!status.has_error()); + + harness.cleanup().await; + logctx.cleanup_successful(); + } } #[cfg(test)] @@ -1322,4 +1697,13 @@ mod test { &serde_json::to_string_pretty(&schema).unwrap(), ); } + + #[test] + fn test_datasets_schema() { + let schema = schemars::schema_for!(DatasetsConfig); + expectorate::assert_contents( + "../schema/omicron-datasets.json", + &serde_json::to_string_pretty(&schema).unwrap(), + ); + } } diff --git a/update-engine/src/buffer.rs b/update-engine/src/buffer.rs index 14dbbd403b..e3ac02458a 100644 --- a/update-engine/src/buffer.rs +++ b/update-engine/src/buffer.rs @@ -123,6 +123,16 @@ impl EventBuffer { EventBufferSteps::new(&self.event_store) } + /// Iterates over all known steps in the buffer in a recursive fashion. + /// + /// The iterator is depth-first and pre-order (i.e. for nested steps, the + /// parent step is visited before the child steps). + pub fn iter_steps_recursive( + &self, + ) -> impl Iterator)> { + self.event_store.event_map_value_dfs() + } + /// Returns information about the given step, as currently tracked by the /// buffer. pub fn get(&self, step_key: &StepKey) -> Option<&EventBufferStepData> { @@ -1272,6 +1282,14 @@ impl StepStatus { matches!(self, Self::Running { .. }) } + /// For completed steps, return the completion reason, otherwise None. + pub fn completion_reason(&self) -> Option<&CompletionReason> { + match self { + Self::Completed { reason, .. } => Some(reason), + _ => None, + } + } + /// For failed steps, return the failure reason, otherwise None. pub fn failure_reason(&self) -> Option<&FailureReason> { match self { diff --git a/update-engine/src/display/group_display.rs b/update-engine/src/display/group_display.rs index 0e04361ce4..9e75b64757 100644 --- a/update-engine/src/display/group_display.rs +++ b/update-engine/src/display/group_display.rs @@ -12,8 +12,9 @@ use swrite::{swrite, SWrite}; use unicode_width::UnicodeWidthStr; use crate::{ - errors::UnknownReportKey, events::EventReport, EventBuffer, - ExecutionStatus, ExecutionTerminalInfo, StepSpec, TerminalKind, + display::ProgressRatioDisplay, errors::UnknownReportKey, + events::EventReport, EventBuffer, ExecutionStatus, ExecutionTerminalInfo, + StepSpec, TerminalKind, }; use super::{ @@ -309,11 +310,13 @@ impl GroupDisplayStats { }; swrite!(line, "{:>HEADER_WIDTH$} ", header.style(header_style)); - let terminal_count = self.terminal_count(); swrite!( line, - "{terminal_count}/{}: {} running, {} {}", - self.total, + "{}: {} running, {} {}", + ProgressRatioDisplay::current_and_total( + self.terminal_count(), + self.total + ), self.running.style(formatter.styles().meta_style), self.completed.style(formatter.styles().meta_style), "completed".style(formatter.styles().progress_style), diff --git a/update-engine/src/display/line_display_shared.rs b/update-engine/src/display/line_display_shared.rs index 99d66bd06f..e31d36dcd7 100644 --- a/update-engine/src/display/line_display_shared.rs +++ b/update-engine/src/display/line_display_shared.rs @@ -16,7 +16,7 @@ use owo_colors::OwoColorize; use swrite::{swrite, SWrite as _}; use crate::{ - display::StepIndexDisplay, + display::ProgressRatioDisplay, events::{ ProgressCounter, ProgressEvent, ProgressEventKind, StepEvent, StepEventKind, StepInfo, StepOutcome, @@ -634,10 +634,12 @@ fn format_progress_counter(counter: &ProgressCounter) -> String { let percent = (counter.current as f64 / total as f64) * 100.0; // <12.34> is 5 characters wide. let percent_width = 5; - let counter_width = total.to_string().len(); format!( - "{:>percent_width$.2}% ({:>counter_width$}/{} {})", - percent, counter.current, total, counter.units, + "{:>percent_width$.2}% ({} {})", + percent, + ProgressRatioDisplay::current_and_total(counter.current, total) + .padded(true), + counter.units, ) } None => format!("{} {}", counter.current, counter.units), @@ -722,7 +724,7 @@ impl LineDisplayFormatter { swrite!( line, "({}) ", - StepIndexDisplay::new( + ProgressRatioDisplay::index_and_total( ld_step_info.step_info.index, ld_step_info.total_steps ) diff --git a/update-engine/src/display/utils.rs b/update-engine/src/display/utils.rs index 3d529ec8a4..f026fb6a1c 100644 --- a/update-engine/src/display/utils.rs +++ b/update-engine/src/display/utils.rs @@ -8,57 +8,107 @@ use std::fmt; use crate::{AbortReason, EventBuffer, StepSpec}; -/// Given an index and a count of total steps, displays `{current}/{total}`. +/// Given current and total, displays `{current}/{total}`. /// -/// Here: -/// -/// * `current` is `index + 1`. +/// * If the `index_and_total` constructor is called, then `current` is `index +/// + 1`. /// * If `padded` is `true`, `current` is right-aligned and padded with spaces /// to the width of `total`. /// /// # Examples /// /// ``` -/// use update_engine::display::StepIndexDisplay; +/// use update_engine::display::ProgressRatioDisplay; /// -/// let display = StepIndexDisplay::new(0, 8); +/// // 0-based index and total. +/// let display = ProgressRatioDisplay::index_and_total(0 as u64, 8 as u64); /// assert_eq!(display.to_string(), "1/8"); -/// let display = StepIndexDisplay::new(82, 230); -/// assert_eq!(display.to_string(), "83/230"); +/// +/// // 1-based current and total. +/// let display = ProgressRatioDisplay::current_and_total(82 as u64, 230 as u64); +/// assert_eq!(display.to_string(), "82/230"); +/// +/// // With padding. /// let display = display.padded(true); -/// assert_eq!(display.to_string(), " 83/230"); +/// assert_eq!(display.to_string(), " 82/230"); /// ``` #[derive(Debug)] -pub struct StepIndexDisplay { - index: usize, - total: usize, +pub struct ProgressRatioDisplay { + current: u64, + total: u64, padded: bool, } -impl StepIndexDisplay { - /// Create a new `StepIndexDisplay`. +impl ProgressRatioDisplay { + /// Create a new `ProgressRatioDisplay` with current and total values. /// - /// The index is 0-based (i.e. 1 is added to it when it is displayed). - pub fn new(index: usize, total: usize) -> Self { - Self { index, total, padded: false } + /// `current` is considered to be 1-based. For example, "20/80 jobs done". + pub fn current_and_total(current: T, total: T) -> Self { + Self { current: current.to_u64(), total: total.to_u64(), padded: false } } + /// Create a new `ProgressRatioDisplay` with index and total values. + /// + /// The index is 0-based (i.e. 1 is added to it). For example, step index 0 + /// out of 8 total steps is shown as "1/8". + pub fn index_and_total(index: T, total: T) -> Self { + Self { + current: index + .to_u64() + .checked_add(1) + .expect("index can't be u64::MAX"), + total: total.to_u64(), + padded: false, + } + } + + /// If set to true, the current value is padded to the same width as the + /// total. pub fn padded(self, padded: bool) -> Self { Self { padded, ..self } } } -impl fmt::Display for StepIndexDisplay { +impl fmt::Display for ProgressRatioDisplay { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if self.padded { let width = self.total.to_string().len(); - write!(f, "{:>width$}/{}", self.index + 1, self.total) + write!(f, "{:>width$}/{}", self.current, self.total) } else { - write!(f, "{}/{}", self.index + 1, self.total) + write!(f, "{}/{}", self.current, self.total) } } } +/// Trait that abstracts over `usize` and `u64`. +/// +/// There are no `From` implementations between `usize` and `u64`, but we +/// assert below that all the architectures we support are 64-bit. +pub trait ToU64 { + fn to_u64(self) -> u64; +} + +const _: () = { + assert!( + std::mem::size_of::() == std::mem::size_of::(), + "usize and u64 are the same size" + ); +}; + +impl ToU64 for usize { + #[inline] + fn to_u64(self) -> u64 { + self as u64 + } +} + +impl ToU64 for u64 { + #[inline] + fn to_u64(self) -> u64 { + self + } +} + /// Displays the message for an execution abort. /// /// Returned by [`AbortReason::message_display`]. diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs index 96a55667fe..de34391fcc 100644 --- a/wicket/src/ui/panes/update.rs +++ b/wicket/src/ui/panes/update.rs @@ -29,7 +29,7 @@ use ratatui::widgets::{ use ratatui::Frame; use slog::{info, o, Logger}; use tui_tree_widget::{Tree, TreeItem, TreeState}; -use update_engine::display::StepIndexDisplay; +use update_engine::display::ProgressRatioDisplay; use update_engine::{ AbortReason, CompletionReason, ExecutionStatus, FailureReason, StepKey, TerminalKind, WillNotBeRunReason, @@ -1986,7 +1986,7 @@ impl ComponentUpdateListState { status_text.push(Span::styled( format!( " (step {})", - StepIndexDisplay::new( + ProgressRatioDisplay::index_and_total( step_key.index, summary.total_steps, ) @@ -2019,7 +2019,7 @@ impl ComponentUpdateListState { status_text.push(Span::styled( format!( " at step {}", - StepIndexDisplay::new( + ProgressRatioDisplay::index_and_total( info.step_key.index, summary.total_steps, ) @@ -2039,7 +2039,7 @@ impl ComponentUpdateListState { status_text.push(Span::styled( format!( " at step {}", - StepIndexDisplay::new( + ProgressRatioDisplay::index_and_total( info.step_key.index, summary.total_steps, ) diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs index 55b4d61c9a..3f460f1e37 100644 --- a/wicketd/src/http_entrypoints.rs +++ b/wicketd/src/http_entrypoints.rs @@ -82,6 +82,7 @@ impl WicketdApi for WicketdApiImpl { config.update_with_inventory_and_bootstrap_peers( &inventory, &ctx.bootstrap_peers, + &ctx.log, ); Ok(HttpResponseOk((&*config).into())) @@ -101,6 +102,7 @@ impl WicketdApi for WicketdApiImpl { config.update_with_inventory_and_bootstrap_peers( &inventory, &ctx.bootstrap_peers, + &ctx.log, ); config .update(body.into_inner(), ctx.baseboard.as_ref()) diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index 56e83fcd41..46ede25eaa 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -26,6 +26,7 @@ use omicron_common::api::external::AllowedSourceIps; use omicron_common::api::external::SwitchLocation; use once_cell::sync::Lazy; use sled_hardware_types::Baseboard; +use slog::debug; use slog::warn; use std::collections::btree_map; use std::collections::BTreeMap; @@ -115,6 +116,7 @@ impl CurrentRssConfig { &mut self, inventory: &RackV1Inventory, bootstrap_peers: &BootstrapPeers, + log: &slog::Logger, ) { let bootstrap_sleds = bootstrap_peers.sleds(); @@ -126,7 +128,15 @@ impl CurrentRssConfig { return None; } - let state = sp.state.as_ref()?; + let Some(state) = sp.state.as_ref() else { + debug!( + log, + "in update_with_inventory_and_bootstrap_peers, \ + filtering out SP with no state"; + "sp" => ?sp, + ); + return None; + }; let baseboard = Baseboard::new_gimlet( state.serial_number.clone(), state.model.clone(), diff --git a/wicketd/tests/integration_tests/inventory.rs b/wicketd/tests/integration_tests/inventory.rs index ed5ad22d5d..c7057e3adc 100644 --- a/wicketd/tests/integration_tests/inventory.rs +++ b/wicketd/tests/integration_tests/inventory.rs @@ -10,6 +10,7 @@ use super::setup::WicketdTestContext; use gateway_messages::SpPort; use gateway_test_utils::setup as gateway_setup; use sled_hardware_types::Baseboard; +use slog::{info, warn}; use wicket::OutputKind; use wicket_common::inventory::{SpIdentifier, SpType}; use wicket_common::rack_setup::BootstrapSledDescription; @@ -32,13 +33,29 @@ async fn test_inventory() { .into_inner(); match response { GetInventoryResponse::Response { inventory, .. } => { - break inventory - } - GetInventoryResponse::Unavailable => { - // Keep polling wicketd until it receives its first results from MGS. - tokio::time::sleep(Duration::from_millis(100)).await; + // Ensure that the SP state is populated -- if it's not, + // then the `configured-bootstrap-sleds` command below + // might return an empty list. + let sp_state_none: Vec<_> = inventory + .sps + .iter() + .filter(|sp| sp.state.is_none()) + .collect(); + if sp_state_none.is_empty() { + break inventory; + } + + warn!( + wicketd_testctx.log(), + "SP state not yet populated for some SPs, retrying"; + "sps" => ?sp_state_none + ) } + GetInventoryResponse::Unavailable => {} } + + // Keep polling wicketd until it receives its first results from MGS. + tokio::time::sleep(Duration::from_millis(100)).await; } }; let inventory = @@ -46,6 +63,8 @@ async fn test_inventory() { .await .expect("get_inventory completed within 10 seconds"); + info!(wicketd_testctx.log(), "inventory returned"; "inventory" => ?inventory); + // 4 SPs attached to the inventory. assert_eq!(inventory.sps.len(), 4); @@ -70,17 +89,17 @@ async fn test_inventory() { serde_json::from_slice(&stdout).expect("stdout is valid JSON"); // This only tests the case that we get sleds back with no current - // bootstrap IP. This does provide svalue: it check that the command - // exists, accesses data within wicket, and returns it in the schema we - // expect. But it does not test the case where a sled does have a - // bootstrap IP. + // bootstrap IP. This does provide some value: it checks that the + // command exists, accesses data within wicket, and returns it in the + // schema we expect. But it does not test the case where a sled does + // have a bootstrap IP. // // Unfortunately, that's a difficult thing to test today. Wicket gets // that information by enumerating the IPs on the bootstrap network and // reaching out to the bootstrap_agent on them directly to ask them who // they are. Our testing setup does not have a way to provide such an // IP, or run a bootstrap_agent on an IP to respond. We should update - // this test when we do have that capabilitiy. + // this test when we do have that capability. assert_eq!( response, vec![ diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index c2e4408f21..ab1f8b971e 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -68,7 +68,7 @@ itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12.1" } itertools-93f6ce9d446188ac = { package = "itertools", version = "0.10.5" } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.5.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.156", features = ["extra_traits"] } +libc = { version = "0.2.158", features = ["extra_traits"] } log = { version = "0.4.21", default-features = false, features = ["std"] } managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] } memchr = { version = "2.7.2" } @@ -107,9 +107,9 @@ string_cache = { version = "0.8.7" } subtle = { version = "2.5.0" } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.74", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } time = { version = "0.3.36", features = ["formatting", "local-offset", "macros", "parsing"] } -tokio = { version = "1.38.1", features = ["full", "test-util"] } +tokio = { version = "1.39.3", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.11", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } -tokio-stream = { version = "0.1.15", features = ["net"] } +tokio-stream = { version = "0.1.15", features = ["net", "sync"] } tokio-util = { version = "0.7.11", features = ["codec", "io-util"] } toml = { version = "0.7.8" } toml_datetime = { version = "0.6.8", default-features = false, features = ["serde"] } @@ -176,7 +176,7 @@ itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12.1" } itertools-93f6ce9d446188ac = { package = "itertools", version = "0.10.5" } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.5.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.156", features = ["extra_traits"] } +libc = { version = "0.2.158", features = ["extra_traits"] } log = { version = "0.4.21", default-features = false, features = ["std"] } managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] } memchr = { version = "2.7.2" } @@ -217,9 +217,9 @@ syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extr syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.74", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } time = { version = "0.3.36", features = ["formatting", "local-offset", "macros", "parsing"] } time-macros = { version = "0.2.18", default-features = false, features = ["formatting", "parsing"] } -tokio = { version = "1.38.1", features = ["full", "test-util"] } +tokio = { version = "1.39.3", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.11", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } -tokio-stream = { version = "0.1.15", features = ["net"] } +tokio-stream = { version = "0.1.15", features = ["net", "sync"] } tokio-util = { version = "0.7.11", features = ["codec", "io-util"] } toml = { version = "0.7.8" } toml_datetime = { version = "0.6.8", default-features = false, features = ["serde"] } @@ -237,7 +237,7 @@ zeroize = { version = "1.7.0", features = ["std", "zeroize_derive"] } [target.x86_64-unknown-linux-gnu.dependencies] dof = { version = "0.3.0", default-features = false, features = ["des"] } linux-raw-sys = { version = "0.4.13", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "std", "system"] } -mio = { version = "0.8.11", features = ["net", "os-ext"] } +mio = { version = "1.0.2", features = ["net", "os-ext"] } nix = { version = "0.28.0", features = ["feature", "fs", "ioctl", "poll", "signal", "term", "uio"] } once_cell = { version = "1.19.0" } rustix = { version = "0.38.34", features = ["fs", "stdio", "system", "termios"] } @@ -246,35 +246,35 @@ signal-hook-mio = { version = "0.2.4", default-features = false, features = ["su [target.x86_64-unknown-linux-gnu.build-dependencies] dof = { version = "0.3.0", default-features = false, features = ["des"] } linux-raw-sys = { version = "0.4.13", default-features = false, features = ["elf", "errno", "general", "ioctl", "no_std", "std", "system"] } -mio = { version = "0.8.11", features = ["net", "os-ext"] } +mio = { version = "1.0.2", features = ["net", "os-ext"] } nix = { version = "0.28.0", features = ["feature", "fs", "ioctl", "poll", "signal", "term", "uio"] } once_cell = { version = "1.19.0" } rustix = { version = "0.38.34", features = ["fs", "stdio", "system", "termios"] } signal-hook-mio = { version = "0.2.4", default-features = false, features = ["support-v0_8", "support-v1_0"] } [target.x86_64-apple-darwin.dependencies] -mio = { version = "0.8.11", features = ["net", "os-ext"] } +mio = { version = "1.0.2", features = ["net", "os-ext"] } nix = { version = "0.28.0", features = ["feature", "fs", "ioctl", "poll", "signal", "term", "uio"] } once_cell = { version = "1.19.0" } rustix = { version = "0.38.34", features = ["fs", "stdio", "system", "termios"] } signal-hook-mio = { version = "0.2.4", default-features = false, features = ["support-v0_8", "support-v1_0"] } [target.x86_64-apple-darwin.build-dependencies] -mio = { version = "0.8.11", features = ["net", "os-ext"] } +mio = { version = "1.0.2", features = ["net", "os-ext"] } nix = { version = "0.28.0", features = ["feature", "fs", "ioctl", "poll", "signal", "term", "uio"] } once_cell = { version = "1.19.0" } rustix = { version = "0.38.34", features = ["fs", "stdio", "system", "termios"] } signal-hook-mio = { version = "0.2.4", default-features = false, features = ["support-v0_8", "support-v1_0"] } [target.aarch64-apple-darwin.dependencies] -mio = { version = "0.8.11", features = ["net", "os-ext"] } +mio = { version = "1.0.2", features = ["net", "os-ext"] } nix = { version = "0.28.0", features = ["feature", "fs", "ioctl", "poll", "signal", "term", "uio"] } once_cell = { version = "1.19.0" } rustix = { version = "0.38.34", features = ["fs", "stdio", "system", "termios"] } signal-hook-mio = { version = "0.2.4", default-features = false, features = ["support-v0_8", "support-v1_0"] } [target.aarch64-apple-darwin.build-dependencies] -mio = { version = "0.8.11", features = ["net", "os-ext"] } +mio = { version = "1.0.2", features = ["net", "os-ext"] } nix = { version = "0.28.0", features = ["feature", "fs", "ioctl", "poll", "signal", "term", "uio"] } once_cell = { version = "1.19.0" } rustix = { version = "0.38.34", features = ["fs", "stdio", "system", "termios"] } @@ -282,7 +282,7 @@ signal-hook-mio = { version = "0.2.4", default-features = false, features = ["su [target.x86_64-unknown-illumos.dependencies] dof = { version = "0.3.0", default-features = false, features = ["des"] } -mio = { version = "0.8.11", features = ["net", "os-ext"] } +mio = { version = "1.0.2", features = ["net", "os-ext"] } nix = { version = "0.28.0", features = ["feature", "fs", "ioctl", "poll", "signal", "term", "uio"] } once_cell = { version = "1.19.0" } rustix = { version = "0.38.34", features = ["fs", "stdio", "system", "termios"] } @@ -291,7 +291,7 @@ toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", featu [target.x86_64-unknown-illumos.build-dependencies] dof = { version = "0.3.0", default-features = false, features = ["des"] } -mio = { version = "0.8.11", features = ["net", "os-ext"] } +mio = { version = "1.0.2", features = ["net", "os-ext"] } nix = { version = "0.28.0", features = ["feature", "fs", "ioctl", "poll", "signal", "term", "uio"] } once_cell = { version = "1.19.0" } rustix = { version = "0.38.34", features = ["fs", "stdio", "system", "termios"] }