diff --git a/.envrc b/.envrc index c77ac326eb..16ef5819f2 100644 --- a/.envrc +++ b/.envrc @@ -4,3 +4,4 @@ PATH_add out/cockroachdb/bin PATH_add out/clickhouse PATH_add out/dendrite-stub/bin +PATH_add out/mgd/root/opt/oxide/mgd/bin diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index 8791ea2fa6..d8de288239 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -69,7 +69,7 @@ ptime -m timeout 1h cargo test --doc --locked --verbose --no-fail-fast # We expect the seed CRDB to be placed here, so we explicitly remove it so the # rmdir check below doesn't get triggered. Nextest doesn't have support for # teardown scripts so this is the best we've got. -rm -rf "$TEST_TMPDIR/crdb-base" +rm -rf "$TEST_TMPDIR/crdb-base"* # # Make sure that we have left nothing around in $TEST_TMPDIR. The easiest way diff --git a/.github/buildomat/jobs/build-and-test-linux.sh b/.github/buildomat/jobs/build-and-test-linux.sh index f33d1a8cfa..715effd080 100755 --- a/.github/buildomat/jobs/build-and-test-linux.sh +++ b/.github/buildomat/jobs/build-and-test-linux.sh @@ -1,8 +1,8 @@ #!/bin/bash #: -#: name = "build-and-test (ubuntu-20.04)" +#: name = "build-and-test (ubuntu-22.04)" #: variety = "basic" -#: target = "ubuntu-20.04" +#: target = "ubuntu-22.04" #: rust_toolchain = "1.72.1" #: output_rules = [ #: "/var/tmp/omicron_tmp/*", diff --git a/.github/buildomat/jobs/clippy.sh b/.github/buildomat/jobs/clippy.sh index dba1021919..5fd31adb76 100755 --- a/.github/buildomat/jobs/clippy.sh +++ b/.github/buildomat/jobs/clippy.sh @@ -29,3 +29,4 @@ ptime -m bash ./tools/install_builder_prerequisites.sh -y banner clippy ptime -m cargo xtask clippy +ptime -m cargo doc diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index c2579d98ea..ff9b44fc40 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -2,7 +2,7 @@ #: #: name = "helios / deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.23" +#: target = "lab-2.0-opte-0.25" #: output_rules = [ #: "%/var/svc/log/oxide-sled-agent:default.log*", #: "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*", @@ -232,11 +232,11 @@ infra_ip_first = \"$UPLINK_IP\" /^infra_ip_last/c\\ infra_ip_last = \"$UPLINK_IP\" } - /^\\[\\[rack_network_config.uplinks/,/^\$/ { - /^gateway_ip/c\\ -gateway_ip = \"$GATEWAY_IP\" - /^uplink_cidr/c\\ -uplink_cidr = \"$UPLINK_IP/32\" + /^\\[\\[rack_network_config.ports/,/^\$/ { + /^routes/c\\ +routes = \\[{nexthop = \"$GATEWAY_IP\", destination = \"0.0.0.0/0\"}\\] + /^addresses/c\\ +addresses = \\[\"$UPLINK_IP/32\"\\] } " pkg/config-rss.toml diff -u pkg/config-rss.toml{~,} || true diff --git a/.github/buildomat/jobs/host-image.sh b/.github/buildomat/jobs/host-image.sh index ba0b4e1ac3..2f4d146a48 100755 --- a/.github/buildomat/jobs/host-image.sh +++ b/.github/buildomat/jobs/host-image.sh @@ -1,11 +1,12 @@ #!/bin/bash #: -#: name = "helios / build OS image" +#: name = "helios / build OS images" #: variety = "basic" #: target = "helios-2.0" #: rust_toolchain = "1.72.1" #: output_rules = [ -#: "=/work/helios/image/output/os.tar.gz", +#: "=/work/helios/upload/os-host.tar.gz", +#: "=/work/helios/upload/os-trampoline.tar.gz", #: ] #: access_repos = [ #: "oxidecomputer/amd-apcb", @@ -44,14 +45,49 @@ TOP=$PWD source "$TOP/tools/include/force-git-over-https.sh" -# Checkout helios at a pinned commit into /work/helios -git clone https://github.com/oxidecomputer/helios.git /work/helios -cd /work/helios +# Check out helios into /work/helios +HELIOSDIR=/work/helios +git clone https://github.com/oxidecomputer/helios.git "$HELIOSDIR" +cd "$HELIOSDIR" +# Record the branch and commit in the output +git status --branch --porcelain=2 +# Setting BUILD_OS to no makes setup skip repositories we don't need for +# building the OS itself (we are just building an image from already built OS). +BUILD_OS=no gmake setup + +# Commands that "helios-build" would ask us to run (either explicitly or +# implicitly, to avoid an error). +rc=0 +pfexec pkg install -q /system/zones/brand/omicron1/tools || rc=$? +case $rc in + # `man pkg` notes that exit code 4 means no changes were made because + # there is nothing to do; that's fine. Any other exit code is an error. + 0 | 4) ;; + *) exit $rc ;; +esac + +pfexec zfs create -p "rpool/images/$USER" + # TODO: Consider importing zones here too? cd "$TOP" +OUTPUTDIR="$HELIOSDIR/upload" +mkdir "$OUTPUTDIR" + +banner OS ./tools/build-host-image.sh -B \ -S /input/package/work/zones/switch-asic.tar.gz \ - /work/helios \ + "$HELIOSDIR" \ /input/package/work/global-zone-packages.tar.gz + +mv "$HELIOSDIR/image/output/os.tar.gz" "$OUTPUTDIR/os-host.tar.gz" + +banner Trampoline + +./tools/build-host-image.sh -R \ + "$HELIOSDIR" \ + /input/package/work/trampoline-global-zone-packages.tar.gz + +mv "$HELIOSDIR/image/output/os.tar.gz" "$OUTPUTDIR/os-trampoline.tar.gz" + diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 64c087524e..5cfb6fcfd7 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -37,7 +37,7 @@ rustc --version # trampoline global zone images. # COMMIT=$(git rev-parse HEAD) -VERSION="1.0.2-0.ci+git${COMMIT:0:11}" +VERSION="1.0.3-0.ci+git${COMMIT:0:11}" echo "$VERSION" >/work/version.txt ptime -m ./tools/install_builder_prerequisites.sh -yp @@ -71,7 +71,7 @@ tarball_src_dir="$(pwd)/out/versioned" stamp_packages() { for package in "$@"; do # TODO: remove once https://github.com/oxidecomputer/omicron-package/pull/54 lands - if [[ $package == maghemite ]]; then + if [[ $package == mg-ddm-gz ]]; then echo "0.0.0" > VERSION tar rvf "out/$package.tar" VERSION rm VERSION @@ -90,7 +90,7 @@ ptime -m cargo run --locked --release --bin omicron-package -- \ -t host target create -i standard -m gimlet -s asic -r multi-sled ptime -m cargo run --locked --release --bin omicron-package -- \ -t host package -stamp_packages omicron-sled-agent maghemite propolis-server overlay +stamp_packages omicron-sled-agent mg-ddm-gz propolis-server overlay # Create global zone package @ /work/global-zone-packages.tar.gz ptime -m ./tools/build-global-zone-packages.sh "$tarball_src_dir" /work @@ -135,7 +135,7 @@ ptime -m cargo run --locked --release --bin omicron-package -- \ -t recovery target create -i trampoline ptime -m cargo run --locked --release --bin omicron-package -- \ -t recovery package -stamp_packages installinator maghemite +stamp_packages installinator mg-ddm-gz # Create trampoline global zone package @ /work/trampoline-global-zone-packages.tar.gz ptime -m ./tools/build-trampoline-global-zone-packages.sh "$tarball_src_dir" /work diff --git a/.github/buildomat/jobs/trampoline-image.sh b/.github/buildomat/jobs/trampoline-image.sh deleted file mode 100755 index 6014d7dca0..0000000000 --- a/.github/buildomat/jobs/trampoline-image.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -#: -#: name = "helios / build trampoline OS image" -#: variety = "basic" -#: target = "helios-2.0" -#: rust_toolchain = "1.72.1" -#: output_rules = [ -#: "=/work/helios/image/output/os.tar.gz", -#: ] -#: access_repos = [ -#: "oxidecomputer/amd-apcb", -#: "oxidecomputer/amd-efs", -#: "oxidecomputer/amd-firmware", -#: "oxidecomputer/amd-flash", -#: "oxidecomputer/amd-host-image-builder", -#: "oxidecomputer/boot-image-tools", -#: "oxidecomputer/chelsio-t6-roms", -#: "oxidecomputer/compliance-pilot", -#: "oxidecomputer/facade", -#: "oxidecomputer/helios", -#: "oxidecomputer/helios-omicron-brand", -#: "oxidecomputer/helios-omnios-build", -#: "oxidecomputer/helios-omnios-extra", -#: "oxidecomputer/nanobl-rs", -#: ] -#: -#: [dependencies.package] -#: job = "helios / package" -#: -#: [[publish]] -#: series = "image" -#: name = "os-trampoline.tar.gz" -#: from_output = "/work/helios/image/output/os.tar.gz" -#: - -set -o errexit -set -o pipefail -set -o xtrace - -cargo --version -rustc --version - -TOP=$PWD - -source "$TOP/tools/include/force-git-over-https.sh" - -# Checkout helios at a pinned commit into /work/helios -git clone https://github.com/oxidecomputer/helios.git /work/helios -cd /work/helios - -cd "$TOP" -./tools/build-host-image.sh -R \ - /work/helios \ - /input/package/work/trampoline-global-zone-packages.tar.gz diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index e169bebff6..29cf7fa85e 100644 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -19,10 +19,22 @@ #: job = "helios / package" #: #: [dependencies.host] -#: job = "helios / build OS image" +#: job = "helios / build OS images" #: -#: [dependencies.trampoline] -#: job = "helios / build trampoline OS image" +#: [[publish]] +#: series = "rot-all" +#: name = "repo.zip.parta" +#: from_output = "/work/repo-rot-all.zip.parta" +#: +#: [[publish]] +#: series = "rot-all" +#: name = "repo.zip.partb" +#: from_output = "/work/repo-rot-all.zip.partb" +#: +#: [[publish]] +#: series = "rot-all" +#: name = "repo.zip.sha256.txt" +#: from_output = "/work/repo-rot-all.zip.sha256.txt" #: #: [[publish]] #: series = "rot-prod-rel" @@ -124,7 +136,7 @@ name = "$kind" version = "$VERSION" [artifact.$kind.source] kind = "file" -path = "/input/$kind/work/helios/image/output/os.tar.gz" +path = "/input/host/work/helios/upload/os-$kind.tar.gz" EOF done @@ -168,6 +180,38 @@ caboose_util_rot() { } SERIES_LIST=() + +# Create an initial `manifest-rot-all.toml` containing the SP images for all +# boards. While we still need to build multiple TUF repos, +# `add_hubris_artifacts` below will append RoT images to this manifest (in +# addition to the single-RoT manifest it creates). +prep_rot_all_series() { + series="rot-all" + + SERIES_LIST+=("$series") + + manifest=/work/manifest-$series.toml + cp /work/manifest.toml "$manifest" + + for board_rev in "${ALL_BOARDS[@]}"; do + board=${board_rev%-?} + tufaceous_board=${board//sidecar/switch} + sp_image="/work/hubris/${board_rev}.zip" + sp_caboose_version=$(/work/caboose-util read-version "$sp_image") + sp_caboose_board=$(/work/caboose-util read-board "$sp_image") + + cat >>"$manifest" <>"$manifest_rot_all" <oxidecomputer/renovate-config", "local>oxidecomputer/renovate-config//rust/autocreate", "local>oxidecomputer/renovate-config:post-upgrade", - "helpers:pinGitHubActionDigests" + "local>oxidecomputer/renovate-config//actions/pin" ] } diff --git a/.github/workflows/check-opte-ver.yml b/.github/workflows/check-opte-ver.yml index a8e18f080e..9fc390277b 100644 --- a/.github/workflows/check-opte-ver.yml +++ b/.github/workflows/check-opte-ver.yml @@ -9,7 +9,7 @@ jobs: check-opte-ver: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Install jq run: sudo apt-get install -y jq - name: Install toml-cli diff --git a/.github/workflows/check-workspace-deps.yml b/.github/workflows/check-workspace-deps.yml index 521afa7359..7ba0c66566 100644 --- a/.github/workflows/check-workspace-deps.yml +++ b/.github/workflows/check-workspace-deps.yml @@ -10,6 +10,6 @@ jobs: check-workspace-deps: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Check Workspace Dependencies run: cargo xtask check-workspace-deps diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index df4cbc9b59..4dc6578b7f 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -17,12 +17,12 @@ jobs: env: RUSTFLAGS: -D warnings steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@e659bf85ee986e37e35cc1c53bfeebe044d8133e # v2 + uses: taiki-e/install-action@f860c89ccbfa08ae7bc92502c27ab631f48b8f9d # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 873b316e16..8cc98f192f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -9,7 +9,7 @@ jobs: check-style: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - name: Report cargo version run: cargo --version - name: Report rustfmt version @@ -27,7 +27,7 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - uses: Swatinem/rust-cache@6fd3edff6979b79f87531400ad694fb7f2c84b1f # v2.2.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version @@ -53,7 +53,7 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - uses: Swatinem/rust-cache@6fd3edff6979b79f87531400ad694fb7f2c84b1f # v2.2.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version @@ -79,7 +79,7 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - uses: Swatinem/rust-cache@6fd3edff6979b79f87531400ad694fb7f2c84b1f # v2.2.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version diff --git a/.github/workflows/update-dendrite.yml b/.github/workflows/update-dendrite.yml index 10d8ef7618..9d79dfc8f9 100644 --- a/.github/workflows/update-dendrite.yml +++ b/.github/workflows/update-dendrite.yml @@ -29,7 +29,7 @@ jobs: steps: # Checkout both the target and integration branches - - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: token: ${{ inputs.reflector_access_token }} fetch-depth: 0 diff --git a/.github/workflows/update-maghemite.yml b/.github/workflows/update-maghemite.yml index 7aa2b8b6c8..e2512dc6ce 100644 --- a/.github/workflows/update-maghemite.yml +++ b/.github/workflows/update-maghemite.yml @@ -29,7 +29,7 @@ jobs: steps: # Checkout both the target and integration branches - - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 with: token: ${{ inputs.reflector_access_token }} fetch-depth: 0 diff --git a/.github/workflows/validate-openapi-spec.yml b/.github/workflows/validate-openapi-spec.yml index 1d6c152296..2716c0571f 100644 --- a/.github/workflows/validate-openapi-spec.yml +++ b/.github/workflows/validate-openapi-spec.yml @@ -10,8 +10,8 @@ jobs: format: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0 - - uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2 with: node-version: '18' - name: Install our tools diff --git a/.gitignore b/.gitignore index 574e867c02..1d7177320f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ core *.vdev debug.out rusty-tags.vi +*.sw* +tags diff --git a/Cargo.lock b/Cargo.lock index e24eec38a5..fba96d19e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,9 +40,9 @@ dependencies = [ [[package]] name = "aes-gcm" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209b47e8954a928e1d72e86eca7000ebb6655fe1436d33eefc2201cad027e237" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ "aead", "aes", @@ -331,9 +331,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", @@ -439,9 +439,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "base64ct" @@ -484,9 +484,9 @@ dependencies = [ [[package]] name = "bcs" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd3ffe8b19a604421a5d461d4a70346223e535903fbc3067138bddbebddcf77" +checksum = "85b6598a2f5d564fb7855dc6b06fd1c38cff5a72bd8b863a4d021938497b440a" dependencies = [ "serde", "thiserror", @@ -495,7 +495,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "bhyve_api_sys", "libc", @@ -505,7 +505,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "libc", "strum", @@ -765,9 +765,9 @@ checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -838,9 +838,9 @@ dependencies = [ [[package]] name = "cancel-safe-futures" -version = "0.1.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84f4df1b54bc954b71be3baaa0771d7fbfebc0b291d2875892f49287f3b5c73d" +checksum = "97eb3167cc49e8a4c1be907817009850b8b9693b225092d0f9b00fa9de076e4e" dependencies = [ "futures-core", "futures-sink", @@ -866,7 +866,7 @@ checksum = "fb9ac64500cc83ce4b9f8dafa78186aa008c8dea77a09b94cd307fd0cd5022a8" dependencies = [ "camino", "cargo-platform", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "thiserror", @@ -1037,23 +1037,6 @@ dependencies = [ "vec_map", ] -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "atty", - "bitflags 1.3.2", - "clap_derive 3.2.25", - "clap_lex 0.2.4", - "indexmap 1.9.3", - "once_cell", - "strsim 0.10.0", - "termcolor", - "textwrap 0.16.0", -] - [[package]] name = "clap" version = "4.4.3" @@ -1061,7 +1044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" dependencies = [ "clap_builder", - "clap_derive 4.4.2", + "clap_derive", ] [[package]] @@ -1072,24 +1055,11 @@ checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", - "clap_lex 0.5.1", + "clap_lex", "strsim 0.10.0", "terminal_size", ] -[[package]] -name = "clap_derive" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" -dependencies = [ - "heck 0.4.1", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "clap_derive" version = "4.4.2" @@ -1102,15 +1072,6 @@ dependencies = [ "syn 2.0.32", ] -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - [[package]] name = "clap_lex" version = "0.5.1" @@ -1235,7 +1196,7 @@ dependencies = [ [[package]] name = "cpuid_profile_config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "propolis", "serde", @@ -1443,13 +1404,13 @@ dependencies = [ [[package]] name = "crucible" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "aes-gcm-siv", "anyhow", "async-recursion", "async-trait", - "base64 0.21.4", + "base64 0.21.5", "bytes", "chrono", "crucible-client-types", @@ -1488,7 +1449,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "anyhow", "chrono", @@ -1504,9 +1465,9 @@ dependencies = [ [[package]] name = "crucible-client-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "crucible-workspace-hack", "schemars", "serde", @@ -1517,7 +1478,7 @@ dependencies = [ [[package]] name = "crucible-common" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "anyhow", "atty", @@ -1545,7 +1506,7 @@ dependencies = [ [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "anyhow", "chrono", @@ -1562,7 +1523,7 @@ dependencies = [ [[package]] name = "crucible-protocol" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "anyhow", "bincode", @@ -1579,7 +1540,7 @@ dependencies = [ [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=657d985247b41e38aac2e271c7ce8bc9ea81f4b6#657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" dependencies = [ "crucible-workspace-hack", "libc", @@ -1762,10 +1723,11 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "datatest-stable" -version = "0.1.3" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eaf86e44e9f0a21f6e42d8e7f83c9ee049f081745eeed1c6f47a613c76e5977" +checksum = "22a384d02609f0774f4dbf0c38fc57eb2769b24c30b9185911ff657ec14837da" dependencies = [ + "camino", "libtest-mimic", "regex", "walkdir", @@ -1904,9 +1866,9 @@ dependencies = [ [[package]] name = "diesel" -version = "2.1.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98235fdc2f355d330a8244184ab6b4b33c28679c0b4158f63138e51d6cf7e88" +checksum = "2268a214a6f118fce1838edba3d1561cf0e78d8de785475957a580a7f8c69d33" dependencies = [ "bitflags 2.4.0", "byteorder", @@ -2022,14 +1984,14 @@ dependencies = [ [[package]] name = "display-error-chain" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e1a8646b2c125eeb9a84ef0faa6d2d102ea0d5da60b824ade2743263117b848" +checksum = "f77af9e75578c1ab34f5f04545a8b05be0c36fbd7a9bb3cf2d2a971e435fdbb9" [[package]] name = "dladm" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "libc", "strum", @@ -2173,7 +2135,7 @@ source = "git+https://github.com/oxidecomputer/dropshot?branch=main#fa728d079708 dependencies = [ "async-stream", "async-trait", - "base64 0.21.4", + "base64 0.21.5", "bytes", "camino", "chrono", @@ -2328,7 +2290,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "base64 0.21.4", + "base64 0.21.5", "camino", "chrono", "futures", @@ -2520,9 +2482,9 @@ checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499" [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -2642,9 +2604,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -2657,9 +2619,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -2667,15 +2629,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -2684,15 +2646,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", @@ -2701,15 +2663,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-timer" @@ -2719,9 +2681,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -2773,7 +2735,7 @@ dependencies = [ name = "gateway-client" version = "0.1.0" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "chrono", "omicron-workspace-hack", "progenitor", @@ -2789,7 +2751,7 @@ dependencies = [ [[package]] name = "gateway-messages" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/management-gateway-service?rev=1e180ae55e56bd17af35cb868ffbd18ce487351d#1e180ae55e56bd17af35cb868ffbd18ce487351d" +source = "git+https://github.com/oxidecomputer/management-gateway-service?rev=2739c18e80697aa6bc235c935176d14b4d757ee9#2739c18e80697aa6bc235c935176d14b4d757ee9" dependencies = [ "bitflags 1.3.2", "hubpack 0.1.2", @@ -2797,14 +2759,15 @@ dependencies = [ "serde_repr", "smoltcp 0.9.1", "static_assertions", + "strum_macros 0.25.2", "uuid", - "zerocopy 0.6.3", + "zerocopy 0.6.4", ] [[package]] name = "gateway-sp-comms" version = "0.1.1" -source = "git+https://github.com/oxidecomputer/management-gateway-service?rev=1e180ae55e56bd17af35cb868ffbd18ce487351d#1e180ae55e56bd17af35cb868ffbd18ce487351d" +source = "git+https://github.com/oxidecomputer/management-gateway-service?rev=2739c18e80697aa6bc235c935176d14b4d757ee9#2739c18e80697aa6bc235c935176d14b4d757ee9" dependencies = [ "async-trait", "backoff", @@ -2817,10 +2780,11 @@ dependencies = [ "lru-cache", "nix 0.26.2 (git+https://github.com/jgallagher/nix?branch=r0.26-illumos)", "once_cell", + "paste", "serde", "serde-big-array 0.5.1", "slog", - "socket2 0.5.3", + "socket2 0.5.4", "string_cache", "thiserror", "tlvc 0.3.1 (git+https://github.com/oxidecomputer/tlvc.git?branch=main)", @@ -3024,7 +2988,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "bytes", "headers-core", "http", @@ -3249,7 +3213,7 @@ dependencies = [ [[package]] name = "hubtools" version = "0.4.1" -source = "git+https://github.com/oxidecomputer/hubtools.git?branch=main#2481445b80f8476041f62a1c8b6301e4918c63ed" +source = "git+https://github.com/oxidecomputer/hubtools.git?branch=main#73cd5a84689d59ecce9da66ad4389c540d315168" dependencies = [ "lpc55_areas", "lpc55_sign", @@ -3261,7 +3225,7 @@ dependencies = [ "tlvc-text", "toml 0.7.8", "x509-cert", - "zerocopy 0.6.3", + "zerocopy 0.6.4", "zip", ] @@ -3297,9 +3261,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", @@ -3406,7 +3370,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" [[package]] name = "illumos-utils" @@ -3418,6 +3382,7 @@ dependencies = [ "byteorder", "camino", "cfg-if 1.0.0", + "crucible-smf", "futures", "ipnetwork", "libc", @@ -3724,7 +3689,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.3", + "socket2 0.5.4", "widestring", "windows-sys 0.48.0", "winreg", @@ -3827,7 +3792,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "quote", "syn 1.0.109", @@ -3963,11 +3928,11 @@ dependencies = [ [[package]] name = "libtest-mimic" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79529479c298f5af41375b0c1a77ef670d450b4c9cd7949d2b43af08121b20ec" +checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7" dependencies = [ - "clap 3.2.25", + "clap 4.4.3", "termcolor", "threadpool", ] @@ -4068,7 +4033,7 @@ dependencies = [ "sha2", "thiserror", "x509-cert", - "zerocopy 0.6.3", + "zerocopy 0.6.4", ] [[package]] @@ -4158,6 +4123,29 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mg-admin-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "either", + "omicron-common 0.1.0", + "omicron-workspace-hack", + "omicron-zone-package", + "progenitor", + "progenitor-client", + "quote", + "reqwest", + "rustfmt-wrapper", + "serde", + "serde_json", + "sled-hardware", + "slog", + "thiserror", + "tokio", + "toml 0.7.8", +] + [[package]] name = "mime" version = "0.3.17" @@ -4362,7 +4350,7 @@ dependencies = [ "rand 0.8.5", "ref-cast", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "sled-agent-client", @@ -4381,7 +4369,7 @@ dependencies = [ "async-bb8-diesel", "async-trait", "authz-macros", - "base64 0.21.4", + "base64 0.21.5", "bb8", "camino", "chrono", @@ -4431,7 +4419,7 @@ dependencies = [ "ref-cast", "regex", "reqwest", - "ring", + "ring 0.16.20", "rustls", "samael", "serde", @@ -4440,6 +4428,7 @@ dependencies = [ "serde_with", "sled-agent-client", "slog", + "static_assertions", "steno", "strum", "subprocess", @@ -4555,7 +4544,7 @@ version = "0.1.0" dependencies = [ "anyhow", "api_identity 0.1.0", - "base64 0.21.4", + "base64 0.21.5", "chrono", "dns-service-client 0.1.0", "futures", @@ -4905,9 +4894,9 @@ dependencies = [ "rand 0.8.5", "regress", "reqwest", - "ring", + "ring 0.16.20", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_derive", "serde_human_bytes", @@ -4947,9 +4936,9 @@ dependencies = [ "progenitor", "rand 0.8.5", "reqwest", - "ring", + "ring 0.16.20", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_derive", "serde_human_bytes", @@ -5019,9 +5008,9 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "base64 0.21.5", "ciborium", "clap 4.4.3", - "crucible-smf", "dropshot", "expectorate", "futures", @@ -5031,6 +5020,7 @@ dependencies = [ "hex", "http", "hyper", + "illumos-utils", "ipcc-key-value", "omicron-common 0.1.0", "omicron-test-utils", @@ -5065,7 +5055,7 @@ dependencies = [ "assert_matches", "async-bb8-diesel", "async-trait", - "base64 0.21.4", + "base64 0.21.5", "bb8", "camino", "cancel-safe-futures", @@ -5097,6 +5087,7 @@ dependencies = [ "itertools 0.11.0", "lazy_static", "macaddr", + "mg-admin-client", "mime_guess", "newtype_derive", "nexus-db-model", @@ -5140,11 +5131,11 @@ dependencies = [ "ref-cast", "regex", "reqwest", - "ring", + "ring 0.16.20", "rustls", "samael", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "serde_urlencoded", @@ -5231,8 +5222,8 @@ dependencies = [ "petgraph", "rayon", "reqwest", - "ring", - "semver 1.0.18", + "ring 0.16.20", + "semver 1.0.20", "serde", "serde_derive", "sled-hardware", @@ -5292,7 +5283,7 @@ dependencies = [ "anyhow", "assert_matches", "async-trait", - "base64 0.21.4", + "base64 0.21.5", "bincode", "bootstore", "bootstrap-agent-client", @@ -5334,7 +5325,6 @@ dependencies = [ "openapi-lint", "openapiv3", "opte-ioctl", - "oxide-vpc", "oximeter 0.1.0", "oximeter-producer 0.1.0", "percent-encoding", @@ -5346,7 +5336,7 @@ dependencies = [ "rcgen", "reqwest", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "serial_test", @@ -5395,7 +5385,7 @@ dependencies = [ "rcgen", "regex", "reqwest", - "ring", + "ring 0.16.20", "rustls", "slog", "subprocess", @@ -5476,11 +5466,12 @@ dependencies = [ "regex-automata 0.3.8", "regex-syntax 0.7.5", "reqwest", - "ring", + "ring 0.16.20", "rustix 0.38.9", "schemars", - "semver 1.0.18", + "semver 1.0.20", "serde", + "serde_json", "sha2", "signature 2.1.0", "similar", @@ -5490,7 +5481,6 @@ dependencies = [ "subtle", "syn 1.0.109", "syn 2.0.32", - "textwrap 0.16.0", "time", "time-macros", "tokio", @@ -5524,7 +5514,7 @@ dependencies = [ "flate2", "futures-util", "reqwest", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_derive", "tar", @@ -5623,7 +5613,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "cfg-if 0.1.10", "dyn-clone", @@ -5634,13 +5624,13 @@ dependencies = [ "serde", "smoltcp 0.8.2", "version_check", - "zerocopy 0.6.3", + "zerocopy 0.6.4", ] [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "cfg-if 0.1.10", "illumos-sys-hdrs", @@ -5653,7 +5643,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "libc", "libnet", @@ -5670,17 +5660,11 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "os_str_bytes" -version = "6.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" - [[package]] name = "oso" -version = "0.26.4" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a07fa8e90aadd02f1d996e94b58cb57142adfc56daf97c2fd783eefaa3f0fe" +checksum = "fceecc04a9e9dcb63a42d937a4249557da8d2695cf83eb5ee78015473ab12ae2" dependencies = [ "impl-trait-for-tuples", "lazy_static", @@ -5693,9 +5677,9 @@ dependencies = [ [[package]] name = "oso-derive" -version = "0.26.4" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "354a3805eaebf00d20dd8446b3ae6fc8f34db013e1d3c1f3f6cd53db17fa6c2d" +checksum = "1766857f83748ce5596ab98e1a57d64ccfe3259e71b7b53289c8c32c2cfef9a8" dependencies = [ "quote", "syn 1.0.109", @@ -5712,7 +5696,7 @@ name = "oxide-client" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.21.4", + "base64 0.21.5", "chrono", "futures", "http", @@ -5733,14 +5717,14 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=631c2017f19cafb1535f621e9e5aa9198ccad869#631c2017f19cafb1535f621e9e5aa9198ccad869" +source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" dependencies = [ "cfg-if 0.1.10", "illumos-sys-hdrs", "opte", "serde", "smoltcp 0.8.2", - "zerocopy 0.6.3", + "zerocopy 0.6.4", ] [[package]] @@ -6339,9 +6323,9 @@ dependencies = [ [[package]] name = "polar-core" -version = "0.26.4" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3833746ee893099f2dff25267b8210394b63983525edb2fe5374bccfb2f77eb" +checksum = "9d1b77e852bec994296c8a1dddc231ab3f112bfa0a0399fc8a7fd8bddfb46b4e" dependencies = [ "indoc 1.0.9", "js-sys", @@ -6404,7 +6388,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "byteorder", "bytes", "fallible-iterator", @@ -6638,7 +6622,7 @@ dependencies = [ [[package]] name = "propolis" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "anyhow", "bhyve_api", @@ -6671,17 +6655,17 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "async-trait", - "base64 0.21.4", + "base64 0.21.5", "crucible-client-types", "futures", "progenitor", "propolis_types", "rand 0.8.5", "reqwest", - "ring", + "ring 0.16.20", "schemars", "serde", "serde_json", @@ -6695,12 +6679,12 @@ dependencies = [ [[package]] name = "propolis-server" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "anyhow", "async-trait", "atty", - "base64 0.21.4", + "base64 0.21.5", "bit_field", "bitvec", "bytes", @@ -6747,7 +6731,7 @@ dependencies = [ [[package]] name = "propolis-server-config" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "cpuid_profile_config", "serde", @@ -6759,7 +6743,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "schemars", "serde", @@ -6964,9 +6948,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -6974,14 +6958,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -6991,7 +6973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", - "ring", + "ring 0.16.20", "time", "yasna", ] @@ -7146,7 +7128,7 @@ version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "bytes", "encoding_rs", "futures-core", @@ -7219,11 +7201,25 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +dependencies = [ + "cc", + "getrandom 0.2.10", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "ringbuffer" version = "0.15.0" @@ -7247,7 +7243,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "bitflags 2.4.0", "serde", "serde_derive", @@ -7428,7 +7424,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5885493fdf0be6cdff808d1533ce878d21cfa49c7086fa00c66355cd9141bfc" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "blake2b_simd", "constant_time_eq 0.3.0", "crossbeam-utils", @@ -7461,7 +7457,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.18", + "semver 1.0.20", ] [[package]] @@ -7506,12 +7502,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" dependencies = [ "log", - "ring", + "ring 0.17.5", "rustls-webpki", "sct", ] @@ -7534,17 +7530,17 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", ] [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -7605,7 +7601,7 @@ name = "samael" version = "0.0.10" source = "git+https://github.com/njaremko/samael?branch=master#52028e45d11ceb7114bf0c730a9971207e965602" dependencies = [ - "base64 0.21.4", + "base64 0.21.5", "bindgen", "chrono", "data-encoding", @@ -7692,8 +7688,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -7745,9 +7741,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" dependencies = [ "serde", ] @@ -8003,9 +7999,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if 1.0.0", "cpufeatures", @@ -8383,9 +8379,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys 0.48.0", @@ -8965,7 +8961,7 @@ source = "git+https://github.com/oxidecomputer/tlvc.git?branch=main#e644a21a7ca9 dependencies = [ "byteorder", "crc", - "zerocopy 0.6.3", + "zerocopy 0.6.4", ] [[package]] @@ -8975,7 +8971,7 @@ source = "git+https://github.com/oxidecomputer/tlvc.git#e644a21a7ca973ed31499106 dependencies = [ "byteorder", "crc", - "zerocopy 0.6.3", + "zerocopy 0.6.4", ] [[package]] @@ -8986,7 +8982,7 @@ dependencies = [ "ron 0.8.1", "serde", "tlvc 0.3.1 (git+https://github.com/oxidecomputer/tlvc.git)", - "zerocopy 0.6.3", + "zerocopy 0.6.4", ] [[package]] @@ -9003,9 +8999,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -9015,7 +9011,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2 0.5.4", "tokio-macros", "windows-sys 0.48.0", ] @@ -9061,7 +9057,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.8.5", - "socket2 0.5.3", + "socket2 0.5.4", "tokio", "tokio-util", "whoami", @@ -9114,9 +9110,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -9229,13 +9225,13 @@ dependencies = [ "pem", "percent-encoding", "reqwest", - "ring", + "ring 0.16.20", "serde", "serde_json", "serde_plain", "snafu", "tempfile", - "untrusted", + "untrusted 0.7.1", "url", "walkdir", ] @@ -9433,7 +9429,7 @@ dependencies = [ "omicron-test-utils", "omicron-workspace-hack", "rand 0.8.5", - "ring", + "ring 0.16.20", "serde", "serde_json", "serde_path_to_error", @@ -9608,9 +9604,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unicode-xid" @@ -9640,6 +9636,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "update-engine" version = "0.1.0" @@ -9799,7 +9801,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "viona_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "libc", "viona_api_sys", @@ -9808,7 +9810,7 @@ dependencies = [ [[package]] name = "viona_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=334df299a56cd0d33e1227ed4ce4d2fe7478d541#334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" dependencies = [ "libc", ] @@ -10042,7 +10044,7 @@ dependencies = [ "ratatui", "reqwest", "rpassword", - "semver 1.0.18", + "semver 1.0.20", "serde", "serde_json", "shell-words", @@ -10107,6 +10109,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "base64 0.21.5", "bootstrap-agent-client", "bytes", "camino", @@ -10135,6 +10138,8 @@ dependencies = [ "installinator-artifact-client", "installinator-artifactd", "installinator-common", + "internal-dns 0.1.0", + "ipnetwork", "itertools 0.11.0", "omicron-certificates", "omicron-common 0.1.0", @@ -10456,12 +10461,12 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3b9c234616391070b0b173963ebc65a9195068e7ed3731c6edac2ec45ebe106" +checksum = "20707b61725734c595e840fb3704378a0cd2b9c74cc9e6e20724838fc6a1e2f9" dependencies = [ "byteorder", - "zerocopy-derive 0.6.3", + "zerocopy-derive 0.6.4", ] [[package]] @@ -10477,9 +10482,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f7f3a471f98d0a61c34322fbbfd10c384b07687f680d4119813713f72308d91" +checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 5173f331a6..c436e3572d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "clients/dpd-client", "clients/gateway-client", "clients/installinator-artifact-client", + "clients/mg-admin-client", "clients/nexus-client", "clients/oxide-client", "clients/oximeter-client", @@ -83,6 +84,7 @@ default-members = [ "clients/oximeter-client", "clients/sled-agent-client", "clients/wicketd-client", + "clients/mg-admin-client", "common", "dev-tools/crdb-seed", "dev-tools/omdb", @@ -139,23 +141,23 @@ approx = "0.5.1" assert_matches = "1.5.0" assert_cmd = "2.0.12" async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "1446f7e0c1f05f33a0581abd51fa873c7652ab61" } -async-trait = "0.1.73" +async-trait = "0.1.74" atomicwrites = "0.4.2" authz-macros = { path = "nexus/authz-macros" } backoff = { version = "0.4.0", features = [ "tokio" ] } -base64 = "0.21.4" +base64 = "0.21.5" bb8 = "0.8.1" -bcs = "0.1.5" +bcs = "0.1.6" bincode = "1.3.3" bootstore = { path = "bootstore" } bootstrap-agent-client = { path = "clients/bootstrap-agent-client" } buf-list = { version = "1.0.3", features = ["tokio1"] } -byteorder = "1.4.3" +byteorder = "1.5.0" bytes = "1.5.0" bytesize = "1.3.0" camino = "1.1" camino-tempfile = "1.0.2" -cancel-safe-futures = "0.1.2" +cancel-safe-futures = "0.1.5" chacha20poly1305 = "0.10.1" ciborium = "0.2.1" cfg-if = "1.0" @@ -165,19 +167,19 @@ cookie = "0.16" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } -crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } +crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } curve25519-dalek = "4" -datatest-stable = "0.1.3" -display-error-chain = "0.1.1" +datatest-stable = "0.2.3" +display-error-chain = "0.2.0" ddm-admin-client = { path = "clients/ddm-admin-client" } db-macros = { path = "nexus/db-macros" } debug-ignore = "1.0.5" derive_more = "0.99.17" derive-where = "1.2.5" -diesel = { version = "2.1.1", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } +diesel = { version = "2.1.3", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } diesel-dtrace = { git = "https://github.com/oxidecomputer/diesel-dtrace", branch = "main" } dns-server = { path = "dns-server" } dns-service-client = { path = "clients/dns-service-client" } @@ -187,14 +189,14 @@ either = "1.9.0" expectorate = "1.1.0" fatfs = "0.3.6" filetime = "0.2.22" -flate2 = "1.0.27" +flate2 = "1.0.28" flume = "0.11.0" foreign-types = "0.3.2" fs-err = "2.9.0" -futures = "0.3.28" +futures = "0.3.29" gateway-client = { path = "clients/gateway-client" } -gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", default-features = false, features = ["std"] } -gateway-sp-comms = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d" } +gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", default-features = false, features = ["std"] } +gateway-sp-comms = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9" } gateway-test-utils = { path = "gateway-test-utils" } glob = "0.3.1" headers = "0.3.9" @@ -208,7 +210,7 @@ httptest = "0.15.4" hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } humantime = "2.1.0" hyper = "0.14" -hyper-rustls = "0.24.1" +hyper-rustls = "0.24.2" hyper-staticfile = "0.9.5" illumos-utils = { path = "illumos-utils" } indexmap = "2.0.0" @@ -229,6 +231,7 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } mime_guess = "2.0.4" mockall = "0.11" newtype_derive = "0.1.6" +mg-admin-client = { path = "clients/mg-admin-client" } nexus-client = { path = "clients/nexus-client" } nexus-db-model = { path = "nexus/db-model" } nexus-db-queries = { path = "nexus/db-queries" } @@ -252,16 +255,16 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.8.3" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "631c2017f19cafb1535f621e9e5aa9198ccad869", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649", features = [ "api", "std" ] } once_cell = "1.18.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "1.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -openssl-probe = "0.1.2" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "631c2017f19cafb1535f621e9e5aa9198ccad869" } -oso = "0.26" +openssl-probe = "0.1.5" +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" } +oso = "0.27" owo-colors = "3.5.0" oximeter = { path = "oximeter/oximeter" } oximeter-client = { path = "clients/oximeter-client" } @@ -270,11 +273,11 @@ oximeter-collector = { path = "oximeter/collector" } oximeter-instruments = { path = "oximeter/instruments" } oximeter-macro-impl = { path = "oximeter/oximeter-macro-impl" } oximeter-producer = { path = "oximeter/producer" } -p256 = "0.11" +p256 = "0.13" parse-display = "0.7.0" partial-io = { version = "0.5.4", features = ["proptest1", "tokio1"] } paste = "1.0.14" -percent-encoding = "2.2.0" +percent-encoding = "2.3.0" pem = "1.1" petgraph = "0.6.4" postgres-protocol = "0.6.6" @@ -284,14 +287,14 @@ pretty-hex = "0.3.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "334df299a56cd0d33e1227ed4ce4d2fe7478d541" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "334df299a56cd0d33e1227ed4ce4d2fe7478d541", features = [ "generated-migration" ] } -propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "334df299a56cd0d33e1227ed4ce4d2fe7478d541", default-features = false, features = ["mock-only"] } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2", features = [ "generated-migration" ] } +propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2", default-features = false, features = ["mock-only"] } proptest = "1.3.1" quote = "1.0" rand = "0.8.5" ratatui = "0.23.0" -rayon = "1.7" +rayon = "1.8" rcgen = "0.10.0" ref-cast = "1.0" regex = "1.9.5" @@ -301,11 +304,11 @@ ring = "0.16" rpassword = "7.2.0" rstest = "0.18.2" rustfmt-wrapper = "0.2" -rustls = "0.21.7" +rustls = "0.21.8" samael = { git = "https://github.com/njaremko/samael", features = ["xmlsec"], branch = "master" } schemars = "0.8.12" secrecy = "0.8.0" -semver = { version = "1.0.18", features = ["std", "serde"] } +semver = { version = "1.0.20", features = ["std", "serde"] } serde = { version = "1.0", default-features = false, features = [ "derive" ] } serde_derive = "1.0" serde_human_bytes = { git = "http://github.com/oxidecomputer/serde_human_bytes", branch = "main" } @@ -315,7 +318,7 @@ serde_tokenstream = "0.2" serde_urlencoded = "0.7.1" serde_with = "2.3.3" serial_test = "0.10" -sha2 = "0.10.7" +sha2 = "0.10.8" sha3 = "0.10.8" shell-words = "1.1.0" signal-hook = "0.3" @@ -354,11 +357,11 @@ textwrap = "0.16.0" test-strategy = "0.2.1" thiserror = "1.0" tofino = { git = "http://github.com/oxidecomputer/tofino", branch = "main" } -tokio = "1.29" +tokio = "1.33.0" tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] } tokio-stream = "0.1.14" tokio-tungstenite = "0.18" -tokio-util = "0.7.8" +tokio-util = "0.7.10" toml = "0.7.8" toml_edit = "0.19.15" topological-sort = "0.2.2" @@ -370,7 +373,7 @@ trust-dns-server = "0.22" trybuild = "1.0.85" tufaceous = { path = "tufaceous" } tufaceous-lib = { path = "tufaceous-lib" } -unicode-width = "0.1.10" +unicode-width = "0.1.11" update-engine = { path = "update-engine" } usdt = "0.3" uuid = { version = "1.4.1", features = ["serde", "v4"] } diff --git a/bootstore/src/schemes/v0/peer.rs b/bootstore/src/schemes/v0/peer.rs index 7d29e2397a..3d273e60eb 100644 --- a/bootstore/src/schemes/v0/peer.rs +++ b/bootstore/src/schemes/v0/peer.rs @@ -91,6 +91,9 @@ pub enum NodeApiRequest { /// These are generated from DDM prefixes learned by the bootstrap agent. PeerAddresses(BTreeSet), + /// Get the local [`SocketAddrV6`] the node is listening on. + GetAddress { responder: oneshot::Sender }, + /// Get the status of this node GetStatus { responder: oneshot::Sender }, @@ -175,6 +178,17 @@ impl NodeHandle { Ok(()) } + /// Get the address of this node + pub async fn get_address(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.tx + .send(NodeApiRequest::GetAddress { responder: tx }) + .await + .map_err(|_| NodeRequestError::Send)?; + let res = rx.await?; + Ok(res) + } + /// Get the status of this node pub async fn get_status(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -361,6 +375,11 @@ impl Node { let mut interval = interval(self.config.time_per_tick); interval.set_missed_tick_behavior(MissedTickBehavior::Delay); let listener = TcpListener::bind(&self.config.addr).await.unwrap(); + // If the config didn't specify a port, let's update it + // with the actual port we binded to on our listener. + if self.config.addr.port() == 0 { + self.config.addr.set_port(listener.local_addr().unwrap().port()); + } while !self.shutdown { tokio::select! { res = listener.accept() => self.on_accept(res).await, @@ -487,6 +506,9 @@ impl Node { info!(self.log, "Updated Peer Addresses: {peers:?}"); self.manage_connections(peers).await; } + NodeApiRequest::GetAddress { responder } => { + let _ = responder.send(self.config.addr); + } NodeApiRequest::GetStatus { responder } => { let status = Status { fsm_ledger_generation: self.fsm_ledger_generation, @@ -1025,11 +1047,11 @@ mod tests { use super::*; use camino_tempfile::Utf8TempDir; use slog::Drain; - use tokio::time::sleep; + use tokio::{task::JoinHandle, time::sleep}; use uuid::Uuid; fn initial_members() -> BTreeSet { - [("a", "1"), ("b", "1"), ("c", "1")] + [("a", "0"), ("b", "1"), ("c", "2")] .iter() .map(|(id, model)| { Baseboard::new_pc(id.to_string(), model.to_string()) @@ -1037,56 +1059,10 @@ mod tests { .collect() } - fn initial_config(tempdir: &Utf8TempDir, port_start: u16) -> Vec { - initial_members() - .into_iter() - .enumerate() - .map(|(i, id)| { - let fsm_file = format!("test-{i}-fsm-state-ledger"); - let network_file = format!("test-{i}-network-config-ledger"); - Config { - id, - addr: format!("[::1]:{}{}", port_start, i).parse().unwrap(), - time_per_tick: Duration::from_millis(20), - learn_timeout: Duration::from_secs(5), - rack_init_timeout: Duration::from_secs(10), - rack_secret_request_timeout: Duration::from_secs(1), - fsm_state_ledger_paths: vec![tempdir - .path() - .join(&fsm_file)], - network_config_ledger_paths: vec![tempdir - .path() - .join(&network_file)], - } - }) - .collect() - } - fn learner_id(n: usize) -> Baseboard { Baseboard::new_pc("learner".to_string(), n.to_string()) } - fn learner_config( - tempdir: &Utf8TempDir, - n: usize, - port_start: u16, - ) -> Config { - let fsm_file = format!("test-learner-{n}-fsm-state-ledger"); - let network_file = format!("test-{n}-network-config-ledger"); - Config { - id: learner_id(n), - addr: format!("[::1]:{}{}", port_start, 3).parse().unwrap(), - time_per_tick: Duration::from_millis(20), - learn_timeout: Duration::from_secs(5), - rack_init_timeout: Duration::from_secs(10), - rack_secret_request_timeout: Duration::from_secs(1), - fsm_state_ledger_paths: vec![tempdir.path().join(&fsm_file)], - network_config_ledger_paths: vec![tempdir - .path() - .join(&network_file)], - } - } - fn log() -> slog::Logger { let decorator = slog_term::PlainDecorator::new(slog_term::TestStdoutWriter); @@ -1095,191 +1071,416 @@ mod tests { slog::Logger::root(drain, o!()) } - #[tokio::test] - async fn basic_3_nodes() { - let port_start = 3333; - let tempdir = Utf8TempDir::new().unwrap(); - let log = log(); - let config = initial_config(&tempdir, port_start); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let (mut node1, handle1) = Node::new(config[1].clone(), &log).await; - let (mut node2, handle2) = Node::new(config[2].clone(), &log).await; - - let jh0 = tokio::spawn(async move { - node0.run().await; - }); - let jh1 = tokio::spawn(async move { - node1.run().await; - }); - let jh2 = tokio::spawn(async move { - node2.run().await; - }); + struct TestNode { + log: Logger, + config: Config, + node_handles: Option<(NodeHandle, JoinHandle<()>)>, + } + + impl TestNode { + fn new(config: Config, log: Logger) -> TestNode { + TestNode { config, log, node_handles: None } + } + + async fn start_node(&mut self) { + // Node must have previously been shutdown (or never started) + assert!( + self.node_handles.is_none(), + "node ({}) already running", + self.config.id + ); + + // Reset port to pick any available + self.config.addr.set_port(0); + + // (Re-)create node with existing config and its persistent state (if any) + let (mut node, handle) = + Node::new(self.config.clone(), &self.log).await; + let jh = tokio::spawn(async move { + node.run().await; + }); + + // Grab assigned port + let port = handle + .get_address() + .await + .unwrap_or_else(|err| { + panic!( + "failed to get local address of node ({}): {err}", + self.config.id + ) + }) + .port(); + self.config.addr.set_port(port); + + self.node_handles = Some((handle, jh)); + } + + async fn shutdown_node(&mut self) { + let (handle, jh) = self.node_handles.take().unwrap_or_else(|| { + panic!("node ({}) not active", self.config.id) + }); + // Signal to the node it should shutdown + handle.shutdown().await.unwrap_or_else(|err| { + panic!("node ({}) failed to shutdown: {err}", self.config.id) + }); + // and wait for its task to spin down. + jh.await.unwrap_or_else(|err| { + panic!("node ({}) task failed: {err}", self.config.id) + }); + } + } + + struct TestNodes { + tempdir: Utf8TempDir, + log: Logger, + nodes: Vec, + learner: Option, + addrs: BTreeSet, + } + + impl TestNodes { + /// Create test nodes for the given set of members. + fn setup(initial_members: BTreeSet) -> TestNodes { + let tempdir = Utf8TempDir::new().unwrap(); + let log = log(); + let nodes = initial_members + .into_iter() + .enumerate() + .map(|(i, id)| { + let fsm_file = format!("test-{i}-fsm-state-ledger"); + let network_file = + format!("test-{i}-network-config-ledger"); + let config = Config { + id, + addr: SocketAddrV6::new( + std::net::Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ), + time_per_tick: Duration::from_millis(20), + learn_timeout: Duration::from_secs(5), + rack_init_timeout: Duration::from_secs(10), + rack_secret_request_timeout: Duration::from_secs(1), + fsm_state_ledger_paths: vec![tempdir + .path() + .join(&fsm_file)], + network_config_ledger_paths: vec![tempdir + .path() + .join(&network_file)], + }; + + TestNode::new(config, log.clone()) + }) + .collect(); + TestNodes { + tempdir, + log, + nodes, + learner: None, // No initial learner node + addrs: BTreeSet::new(), + } + } + + /// (Re-)start the given node and update peer addresses for everyone + async fn start_node(&mut self, i: usize) { + let node = &mut self.nodes[i]; + node.start_node().await; + self.addrs.insert(node.config.addr); + self.load_all_peer_addresses().await; + } + + // Stop the given node and update peer addresses for everyone + async fn shutdown_node(&mut self, i: usize) { + let node = &mut self.nodes[i]; + let addr = node.config.addr; + node.shutdown_node().await; + self.addrs.remove(&addr); + self.load_all_peer_addresses().await; + } + + /// Stop all active nodes (including the learner, if present). + async fn shutdown_all(&mut self) { + let nodes = self + .nodes + .iter_mut() + .chain(&mut self.learner) + .filter(|node| node.node_handles.is_some()); + for node in nodes { + node.shutdown_node().await; + } + self.addrs.clear(); + self.learner = None; + } + + /// Configure new learner node + async fn add_learner(&mut self, n: usize) { + assert!( + self.learner.is_none(), + "learner node already configured ({})", + self.learner.as_ref().unwrap().config.id + ); + + let fsm_file = format!("test-learner-{n}-fsm-state-ledger"); + let network_file = format!("test-{n}-network-config-ledger"); + let config = Config { + id: learner_id(n), + addr: SocketAddrV6::new(std::net::Ipv6Addr::LOCALHOST, 0, 0, 0), + time_per_tick: Duration::from_millis(20), + learn_timeout: Duration::from_secs(5), + rack_init_timeout: Duration::from_secs(10), + rack_secret_request_timeout: Duration::from_secs(1), + fsm_state_ledger_paths: vec![self + .tempdir + .path() + .join(&fsm_file)], + network_config_ledger_paths: vec![self + .tempdir + .path() + .join(&network_file)], + }; + + self.learner = Some(TestNode::new(config, self.log.clone())); + } - // Inform each node about the known addresses - let mut addrs: BTreeSet<_> = config.iter().map(|c| c.addr).collect(); - for handle in [&handle0, &handle1, &handle2] { - let _ = handle.load_peer_addresses(addrs.clone()).await; + /// Start a configured learner node and update peer addresses for everyone + async fn start_learner(&mut self) { + let learner = + self.learner.as_mut().expect("no learner node configured"); + learner.start_node().await; + let learner_addr = learner.config.addr; + + // Inform the learner and other nodes about all addresses including + // the learner. This simulates DDM discovery. + self.addrs.insert(learner_addr); + self.load_all_peer_addresses().await; } + /// Stop the learner node (but leave it configured) and update peer addresses for everyone + /// Can also optionally wipe the ledger persisted on disk. + async fn shutdown_learner(&mut self, wipe_ledger: bool) { + let learner = + self.learner.as_mut().expect("no learner node configured"); + let addr = learner.config.addr; + learner.shutdown_node().await; + + if wipe_ledger { + std::fs::remove_file(&learner.config.fsm_state_ledger_paths[0]) + .expect("failed to remove ledger"); + } + + // Update peer addresses + self.addrs.remove(&addr); + self.load_all_peer_addresses().await; + } + + /// Remove a configured learner node + async fn remove_learner(&mut self) { + // Shutdown the node if it's running + if matches!( + self.learner, + Some(TestNode { node_handles: Some(_), .. }) + ) { + self.shutdown_learner(false).await; + } + let _ = self.learner.take().expect("no learner node configured"); + } + + /// Inform each active node about its peers + async fn load_all_peer_addresses(&self) { + let nodes = + self.nodes.iter().chain(&self.learner).filter_map(|node| { + node.node_handles + .as_ref() + .map(|(h, _)| (&node.config.id, h)) + }); + for (id, node) in nodes { + node.load_peer_addresses(self.addrs.clone()).await.unwrap_or_else(|err| { + panic!("failed to update peer addresses for node ({id}): {err}") + }); + } + } + + /// Returns an iterator that yields the [`NodeHandle`]'s for all active + /// nodes (including the learner node, if present). + fn iter(&self) -> impl Iterator { + self.nodes + .iter() + .chain(&self.learner) + .filter_map(|node| node.node_handles.as_ref().map(|(h, _)| h)) + } + + /// To ensure deterministic learning of shares from node 0 which sorts first + /// we wait to ensure that the learner sees peer0 as connected before we + /// call `init_learner` + /// + /// Panics if the connection doesn't happen within `POLL_TIMEOUT` + async fn wait_for_learner_to_connect_to_node(&self, i: usize) { + const POLL_TIMEOUT: Duration = Duration::from_secs(5); + let start = Instant::now(); + loop { + let timeout = + POLL_TIMEOUT.saturating_sub(Instant::now() - start); + tokio::select! { + _ = sleep(timeout) => { + panic!("Learner not connected to node {i}"); + } + status = self[LEARNER].get_status() => { + let status = status.unwrap(); + let id = &self.nodes[i].config.id; + if status.connections.contains_key(id) { + break; + } + tokio::time::sleep(Duration::from_millis(1)).await; + } + } + } + } + } + + impl std::ops::Index for TestNodes { + type Output = NodeHandle; + + fn index(&self, index: usize) -> &Self::Output { + self.nodes[index] + .node_handles + .as_ref() + .map(|(handle, _)| handle) + .unwrap_or_else(|| panic!("node{index} not running")) + } + } + + // A little convenience to access the learner node in a similar + // manner as other nodes (indexing) but with a non-usize index. + const LEARNER: () = (); + impl std::ops::Index<()> for TestNodes { + type Output = NodeHandle; + + fn index(&self, _: ()) -> &Self::Output { + self.learner + .as_ref() + .expect("no learner node") + .node_handles + .as_ref() + .map(|(handle, _)| handle) + .expect("learner node not running") + } + } + + #[tokio::test] + async fn basic_3_nodes() { + // Create and start test nodes + let mut nodes = TestNodes::setup(initial_members()); + nodes.start_node(0).await; + nodes.start_node(1).await; + nodes.start_node(2).await; + let rack_uuid = RackUuid(Uuid::new_v4()); - handle0.init_rack(rack_uuid, initial_members()).await.unwrap(); + nodes[0].init_rack(rack_uuid, initial_members()).await.unwrap(); - let status = handle0.get_status().await; + let status = nodes[0].get_status().await; println!("Status = {status:?}"); // Ensure we can load the rack secret at all nodes - handle0.load_rack_secret().await.unwrap(); - handle1.load_rack_secret().await.unwrap(); - handle2.load_rack_secret().await.unwrap(); + for node in nodes.iter() { + node.load_rack_secret().await.unwrap(); + } // load the rack secret a second time on node0 - handle0.load_rack_secret().await.unwrap(); + nodes[0].load_rack_secret().await.unwrap(); // Shutdown the node2 and make sure we can still load the rack // secret (threshold=2) at node0 and node1 - handle2.shutdown().await.unwrap(); - jh2.await.unwrap(); - handle0.load_rack_secret().await.unwrap(); - handle1.load_rack_secret().await.unwrap(); - - // Add a learner node - let learner_conf = learner_config(&tempdir, 1, port_start); - let (mut learner, learner_handle) = - Node::new(learner_conf.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); - // Inform the learner and node0 and node1 about all addresses including - // the learner. This simulates DDM discovery - addrs.insert(learner_conf.addr); - let _ = learner_handle.load_peer_addresses(addrs.clone()).await; - let _ = handle0.load_peer_addresses(addrs.clone()).await; - let _ = handle1.load_peer_addresses(addrs.clone()).await; + nodes.shutdown_node(2).await; + nodes[0].load_rack_secret().await.unwrap(); + nodes[1].load_rack_secret().await.unwrap(); + + // Add and start a learner node + nodes.add_learner(1).await; + nodes.start_learner().await; // Tell the learner to go ahead and learn its share. - learner_handle.init_learner().await.unwrap(); + nodes[LEARNER].init_learner().await.unwrap(); // Shutdown node1 and show that we can still load the rack secret at // node0 and the learner, because threshold=2 and it never changes. - handle1.shutdown().await.unwrap(); - jh1.await.unwrap(); - handle0.load_rack_secret().await.unwrap(); - learner_handle.load_rack_secret().await.unwrap(); + nodes.shutdown_node(1).await; + nodes[0].load_rack_secret().await.unwrap(); + nodes[LEARNER].load_rack_secret().await.unwrap(); - // Now shutdown the learner and show that node0 cannot load the rack secret - learner_handle.shutdown().await.unwrap(); - learner_jh.await.unwrap(); - handle0.load_rack_secret().await.unwrap_err(); + // Now shutdown and remove the learner and show that node0 cannot load the rack secret + nodes.remove_learner().await; + nodes[0].load_rack_secret().await.unwrap_err(); - // Reload an node from persistent state and successfully reload the + // Reload a node from persistent state and successfully reload the // rack secret. - let (mut node1, handle1) = Node::new(config[1].clone(), &log).await; - let jh1 = tokio::spawn(async move { - node1.run().await; - }); - let _ = handle1.load_peer_addresses(addrs.clone()).await; - handle0.load_rack_secret().await.unwrap(); + nodes.start_node(1).await; + nodes[0].load_rack_secret().await.unwrap(); - // Add a second learner + // Grab the current generation numbers let peer0_gen = - handle0.get_status().await.unwrap().fsm_ledger_generation; + nodes[0].get_status().await.unwrap().fsm_ledger_generation; let peer1_gen = - handle1.get_status().await.unwrap().fsm_ledger_generation; - let learner_config = learner_config(&tempdir, 2, port_start); - let (mut learner, learner_handle) = - Node::new(learner_config.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); + nodes[1].get_status().await.unwrap().fsm_ledger_generation; + + // Add and start a second learner + nodes.add_learner(2).await; + nodes.start_learner().await; - // Inform the learner, node0, and node1 about all addresses including - // the learner. This simulates DDM discovery - addrs.insert(learner_config.addr); - let _ = learner_handle.load_peer_addresses(addrs.clone()).await; - let _ = handle0.load_peer_addresses(addrs.clone()).await; - let _ = handle1.load_peer_addresses(addrs.clone()).await; + // Wait for the learner to connect to node 0 + nodes.wait_for_learner_to_connect_to_node(0).await; // Tell the learner to go ahead and learn its share. - learner_handle.init_learner().await.unwrap(); + nodes[LEARNER].init_learner().await.unwrap(); // Get the new generation numbers let peer0_gen_new = - handle0.get_status().await.unwrap().fsm_ledger_generation; + nodes[0].get_status().await.unwrap().fsm_ledger_generation; let peer1_gen_new = - handle1.get_status().await.unwrap().fsm_ledger_generation; + nodes[1].get_status().await.unwrap().fsm_ledger_generation; - // Ensure only one of the peers generation numbers gets bumped - assert!( - (peer0_gen_new == peer0_gen && peer1_gen_new == peer1_gen + 1) - || (peer0_gen_new == peer0_gen + 1 - && peer1_gen_new == peer1_gen) - ); + // Ensure only peer 0's generation number gets bumped + assert_eq!(peer0_gen_new, peer0_gen + 1); + assert_eq!(peer1_gen_new, peer1_gen); + + // Now we can stop the learner, wipe its ledger, and restart it. + nodes.shutdown_learner(true).await; + nodes.start_learner().await; // Wipe the learner ledger, restart the learner and instruct it to // relearn its share, and ensure that the neither generation number gets - // bumped because persistence doesn't occur. - learner_handle.shutdown().await.unwrap(); - learner_jh.await.unwrap(); - std::fs::remove_file(&learner_config.fsm_state_ledger_paths[0]) - .unwrap(); - let (mut learner, learner_handle) = - Node::new(learner_config.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); - let _ = learner_handle.load_peer_addresses(addrs.clone()).await; - learner_handle.init_learner().await.unwrap(); + // bumped because persistence doesn't occur. But for that to happen + // we need to make sure the learner asks the same peer, which is node 0 since + // it sorts first based on its id which is of type `Baseboard`. + nodes.wait_for_learner_to_connect_to_node(0).await; + nodes[LEARNER].init_learner().await.unwrap(); + + // Ensure the peers' generation numbers didn't get bumped. The learner + // should've asked the same sled for a share first, which it already + // handed out. let peer0_gen_new_2 = - handle0.get_status().await.unwrap().fsm_ledger_generation; + nodes[0].get_status().await.unwrap().fsm_ledger_generation; let peer1_gen_new_2 = - handle1.get_status().await.unwrap().fsm_ledger_generation; - - // Ensure the peer's generation numbers don't get bumped. The learner - // will ask the same sled for a share first, which it already handed - // out. - assert!( - peer0_gen_new == peer0_gen_new_2 - && peer1_gen_new == peer1_gen_new_2 - ); + nodes[1].get_status().await.unwrap().fsm_ledger_generation; + assert_eq!(peer0_gen_new, peer0_gen_new_2); + assert_eq!(peer1_gen_new, peer1_gen_new_2); - // Shutdown the new learner, node0, and node1 - learner_handle.shutdown().await.unwrap(); - learner_jh.await.unwrap(); - handle0.shutdown().await.unwrap(); - jh0.await.unwrap(); - handle1.shutdown().await.unwrap(); - jh1.await.unwrap(); + // Shut it all down + nodes.shutdown_all().await; } #[tokio::test] async fn network_config() { - let port_start = 4444; - let tempdir = Utf8TempDir::new().unwrap(); - let log = log(); - let config = initial_config(&tempdir, port_start); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let (mut node1, handle1) = Node::new(config[1].clone(), &log).await; - let (mut node2, handle2) = Node::new(config[2].clone(), &log).await; - - let jh0 = tokio::spawn(async move { - node0.run().await; - }); - let jh1 = tokio::spawn(async move { - node1.run().await; - }); - let jh2 = tokio::spawn(async move { - node2.run().await; - }); - - // Inform each node about the known addresses - let mut addrs: BTreeSet<_> = config.iter().map(|c| c.addr).collect(); - for handle in [&handle0, &handle1, &handle2] { - let _ = handle.load_peer_addresses(addrs.clone()).await; - } + // Create and start test nodes + let mut nodes = TestNodes::setup(initial_members()); + nodes.start_node(0).await; + nodes.start_node(1).await; + nodes.start_node(2).await; // Ensure there is no network config at any of the nodes - for handle in [&handle0, &handle1, &handle2] { - assert_eq!(None, handle.get_network_config().await.unwrap()); + for node in nodes.iter() { + assert_eq!(None, node.get_network_config().await.unwrap()); } // Update the network config at node0 and ensure it has taken effect @@ -1287,10 +1488,10 @@ mod tests { generation: 1, blob: b"Some network data".to_vec(), }; - handle0.update_network_config(network_config.clone()).await.unwrap(); + nodes[0].update_network_config(network_config.clone()).await.unwrap(); assert_eq!( Some(&network_config), - handle0.get_network_config().await.unwrap().as_ref() + nodes[0].get_network_config().await.unwrap().as_ref() ); // Poll node1 and node2 until the network config update shows up @@ -1305,13 +1506,13 @@ mod tests { _ = sleep(timeout) => { panic!("Network config not replicated"); } - res = handle1.get_network_config(), if !node1_done => { + res = nodes[1].get_network_config(), if !node1_done => { if res.unwrap().as_ref() == Some(&network_config) { node1_done = true; continue; } } - res = handle2.get_network_config(), if !node2_done => { + res = nodes[2].get_network_config(), if !node2_done => { if res.unwrap().as_ref() == Some(&network_config) { node2_done = true; continue; @@ -1321,18 +1522,8 @@ mod tests { } // Bring a learner online - let learner_conf = learner_config(&tempdir, 1, port_start); - let (mut learner, learner_handle) = - Node::new(learner_conf.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); - // Inform the learner and other nodes about all addresses including - // the learner. This simulates DDM discovery. - addrs.insert(learner_conf.addr); - for handle in [&learner_handle, &handle0, &handle1, &handle2] { - let _ = handle.load_peer_addresses(addrs.clone()).await; - } + nodes.add_learner(1).await; + nodes.start_learner().await; // Poll the learner to ensure it gets the network config // Note that the learner doesn't even need to learn its share @@ -1345,7 +1536,7 @@ mod tests { _ = sleep(timeout) => { panic!("Network config not replicated"); } - res = learner_handle.get_network_config() => { + res = nodes[LEARNER].get_network_config() => { if res.unwrap().as_ref() == Some(&network_config) { done = true; } @@ -1355,34 +1546,26 @@ mod tests { // Stop node0, bring it back online and ensure it still sees the config // at generation 1 - handle0.shutdown().await.unwrap(); - jh0.await.unwrap(); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let jh0 = tokio::spawn(async move { - node0.run().await; - }); + nodes.shutdown_node(0).await; + nodes.start_node(0).await; assert_eq!( Some(&network_config), - handle0.get_network_config().await.unwrap().as_ref() + nodes[0].get_network_config().await.unwrap().as_ref() ); // Stop node0 again, update network config via node1, bring node0 back online, // and ensure all nodes see the latest configuration. + nodes.shutdown_node(0).await; let new_config = NetworkConfig { generation: 2, blob: b"Some more network data".to_vec(), }; - handle0.shutdown().await.unwrap(); - jh0.await.unwrap(); - handle1.update_network_config(new_config.clone()).await.unwrap(); + nodes[1].update_network_config(new_config.clone()).await.unwrap(); assert_eq!( Some(&new_config), - handle1.get_network_config().await.unwrap().as_ref() + nodes[1].get_network_config().await.unwrap().as_ref() ); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let jh0 = tokio::spawn(async move { - node0.run().await; - }); + nodes.start_node(0).await; let start = Instant::now(); // These should all resolve instantly, so no real need for a select, // which is getting tedious. @@ -1392,8 +1575,8 @@ mod tests { if Instant::now() - start > POLL_TIMEOUT { panic!("network config not replicated"); } - for h in [&handle0, &handle1, &handle2, &learner_handle] { - if h.get_network_config().await.unwrap().as_ref() + for node in nodes.iter() { + if node.get_network_config().await.unwrap().as_ref() != Some(&new_config) { // We need to try again @@ -1410,16 +1593,11 @@ mod tests { current_generation: 2, }); assert_eq!( - handle0.update_network_config(network_config).await, + nodes[0].update_network_config(network_config).await, expected ); // Shut it all down - for h in [handle0, handle1, handle2, learner_handle] { - let _ = h.shutdown().await; - } - for jh in [jh0, jh1, jh2, learner_jh] { - jh.await.unwrap(); - } + nodes.shutdown_all().await; } } diff --git a/bootstore/src/schemes/v0/storage.rs b/bootstore/src/schemes/v0/storage.rs index ee31d24f05..327acc6058 100644 --- a/bootstore/src/schemes/v0/storage.rs +++ b/bootstore/src/schemes/v0/storage.rs @@ -5,9 +5,9 @@ //! Storage for the v0 bootstore scheme //! //! We write two pieces of data to M.2 devices in production via -//! [`omicron_common::Ledger`]: +//! [`omicron_common::ledger::Ledger`]: //! -//! 1. [`super::Fsm::State`] for bootstore state itself +//! 1. [`super::State`] for bootstore state itself //! 2. A network config blob required for pre-rack-unlock configuration //! diff --git a/clients/bootstrap-agent-client/src/lib.rs b/clients/bootstrap-agent-client/src/lib.rs index 3f8b20e1f5..19ecb599f3 100644 --- a/clients/bootstrap-agent-client/src/lib.rs +++ b/clients/bootstrap-agent-client/src/lib.rs @@ -20,6 +20,8 @@ progenitor::generate_api!( derives = [schemars::JsonSchema], replace = { Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, } ); diff --git a/clients/ddm-admin-client/build.rs b/clients/ddm-admin-client/build.rs index e3c1345eda..da74ee9962 100644 --- a/clients/ddm-admin-client/build.rs +++ b/clients/ddm-admin-client/build.rs @@ -21,20 +21,21 @@ fn main() -> Result<()> { println!("cargo:rerun-if-changed=../../package-manifest.toml"); let config: Config = toml::from_str(&manifest) - .context("failed to parse ../../package-manifest.toml")?; - let maghemite = config + .context("failed to parse ../package-manifest.toml")?; + + let ddm = config .packages - .get("maghemite") - .context("missing maghemite package in ../../package-manifest.toml")?; + .get("mg-ddm-gz") + .context("missing mg-ddm-gz package in ../package-manifest.toml")?; - let local_path = match &maghemite.source { + let local_path = match &ddm.source { PackageSource::Prebuilt { commit, .. } => { // Report a relatively verbose error if we haven't downloaded the requisite // openapi spec. let local_path = format!("../../out/downloads/ddm-admin-{commit}.json"); if !Path::new(&local_path).exists() { - bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); + bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_ddm_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); } println!("cargo:rerun-if-changed={local_path}"); local_path @@ -51,7 +52,9 @@ fn main() -> Result<()> { } _ => { - bail!("maghemite external package must have type `prebuilt` or `manual`") + bail!( + "mg-ddm external package must have type `prebuilt` or `manual`" + ) } }; diff --git a/clients/mg-admin-client/Cargo.toml b/clients/mg-admin-client/Cargo.toml new file mode 100644 index 0000000000..c444fee32f --- /dev/null +++ b/clients/mg-admin-client/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "mg-admin-client" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[dependencies] +either.workspace = true +progenitor-client.workspace = true +reqwest = { workspace = true, features = ["json", "stream", "rustls-tls"] } +serde.workspace = true +slog.workspace = true +thiserror.workspace = true +tokio.workspace = true +omicron-common.workspace = true +sled-hardware.workspace = true +omicron-workspace-hack.workspace = true + +[build-dependencies] +anyhow.workspace = true +omicron-zone-package.workspace = true +progenitor.workspace = true +quote.workspace = true +rustfmt-wrapper.workspace = true +serde_json.workspace = true +toml.workspace = true diff --git a/clients/mg-admin-client/build.rs b/clients/mg-admin-client/build.rs new file mode 100644 index 0000000000..dcc7ae61cb --- /dev/null +++ b/clients/mg-admin-client/build.rs @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2022 Oxide Computer Company + +use anyhow::bail; +use anyhow::Context; +use anyhow::Result; +use omicron_zone_package::config::Config; +use omicron_zone_package::package::PackageSource; +use quote::quote; +use std::env; +use std::fs; +use std::path::Path; + +fn main() -> Result<()> { + // Find the current maghemite repo commit from our package manifest. + let manifest = fs::read_to_string("../../package-manifest.toml") + .context("failed to read ../../package-manifest.toml")?; + println!("cargo:rerun-if-changed=../../package-manifest.toml"); + + let config: Config = toml::from_str(&manifest) + .context("failed to parse ../../package-manifest.toml")?; + let mg = config + .packages + .get("mgd") + .context("missing mgd package in ../../package-manifest.toml")?; + + let local_path = match &mg.source { + PackageSource::Prebuilt { commit, .. } => { + // Report a relatively verbose error if we haven't downloaded the requisite + // openapi spec. + let local_path = + format!("../../out/downloads/mg-admin-{commit}.json"); + if !Path::new(&local_path).exists() { + bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_mg_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); + } + println!("cargo:rerun-if-changed={local_path}"); + local_path + } + + PackageSource::Manual => { + let local_path = + "../../out/downloads/mg-admin-manual.json".to_string(); + if !Path::new(&local_path).exists() { + bail!("{local_path} doesn't exist, please copy manually built mg-admin.json there!"); + } + println!("cargo:rerun-if-changed={local_path}"); + local_path + } + + _ => { + bail!("mgd external package must have type `prebuilt` or `manual`") + } + }; + + let spec = { + let bytes = fs::read(&local_path) + .with_context(|| format!("failed to read {local_path}"))?; + serde_json::from_slice(&bytes).with_context(|| { + format!("failed to parse {local_path} as openapi spec") + })? + }; + + let code = progenitor::Generator::new( + progenitor::GenerationSettings::new() + .with_inner_type(quote!(slog::Logger)) + .with_pre_hook(quote! { + |log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + } + }) + .with_post_hook(quote! { + |log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + } + }), + ) + .generate_tokens(&spec) + .with_context(|| { + format!("failed to generate progenitor client from {local_path}") + })?; + + let content = rustfmt_wrapper::rustfmt(code).with_context(|| { + format!("rustfmt failed on progenitor code from {local_path}") + })?; + + let out_file = + Path::new(&env::var("OUT_DIR").expect("OUT_DIR env var not set")) + .join("mg-admin-client.rs"); + + fs::write(&out_file, content).with_context(|| { + format!("failed to write client to {}", out_file.display()) + })?; + + Ok(()) +} diff --git a/clients/mg-admin-client/src/lib.rs b/clients/mg-admin-client/src/lib.rs new file mode 100644 index 0000000000..bb1d925c73 --- /dev/null +++ b/clients/mg-admin-client/src/lib.rs @@ -0,0 +1,83 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +#![allow(clippy::redundant_closure_call)] +#![allow(clippy::needless_lifetimes)] +#![allow(clippy::match_single_binding)] +#![allow(clippy::clone_on_copy)] +#![allow(rustdoc::broken_intra_doc_links)] +#![allow(rustdoc::invalid_html_tags)] + +#[allow(dead_code)] +mod inner { + include!(concat!(env!("OUT_DIR"), "/mg-admin-client.rs")); +} + +pub use inner::types; +pub use inner::Error; + +use inner::Client as InnerClient; +use omicron_common::api::external::BgpPeerState; +use slog::Logger; +use std::net::Ipv6Addr; +use std::net::SocketAddr; +use thiserror::Error; + +// TODO-cleanup Is it okay to hardcode this port number here? +const MGD_PORT: u16 = 4676; + +#[derive(Debug, Error)] +pub enum MgError { + #[error("Failed to construct an HTTP client: {0}")] + HttpClient(#[from] reqwest::Error), + + #[error("Failed making HTTP request to mgd: {0}")] + MgApi(#[from] Error), +} + +impl From for BgpPeerState { + fn from(s: inner::types::FsmStateKind) -> BgpPeerState { + use inner::types::FsmStateKind; + match s { + FsmStateKind::Idle => BgpPeerState::Idle, + FsmStateKind::Connect => BgpPeerState::Connect, + FsmStateKind::Active => BgpPeerState::Active, + FsmStateKind::OpenSent => BgpPeerState::OpenSent, + FsmStateKind::OpenConfirm => BgpPeerState::OpenConfirm, + FsmStateKind::SessionSetup => BgpPeerState::SessionSetup, + FsmStateKind::Established => BgpPeerState::Established, + } + } +} + +#[derive(Debug, Clone)] +pub struct Client { + pub inner: InnerClient, + pub log: Logger, +} + +impl Client { + /// Creates a new [`Client`] that points to localhost + pub fn localhost(log: &Logger) -> Result { + Self::new(log, SocketAddr::new(Ipv6Addr::LOCALHOST.into(), MGD_PORT)) + } + + pub fn new(log: &Logger, mgd_addr: SocketAddr) -> Result { + let dur = std::time::Duration::from_secs(60); + let log = log.new(slog::o!("MgAdminClient" => mgd_addr)); + + let inner = reqwest::ClientBuilder::new() + .connect_timeout(dur) + .timeout(dur) + .build()?; + let inner = InnerClient::new_with_client( + &format!("http://{mgd_addr}"), + inner, + log.clone(), + ); + Ok(Self { inner, log }) + } +} diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs index 33a68cb3ce..23ceb114fc 100644 --- a/clients/nexus-client/src/lib.rs +++ b/clients/nexus-client/src/lib.rs @@ -23,6 +23,8 @@ progenitor::generate_api!( }), replace = { Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, MacAddr = omicron_common::api::external::MacAddr, Name = omicron_common::api::external::Name, NewPasswordHash = omicron_passwords::NewPasswordHash, diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 3daac7dd60..0df21d894e 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -5,11 +5,33 @@ //! Interface for making API requests to a Sled Agent use async_trait::async_trait; -use omicron_common::generate_logging_api; use std::convert::TryFrom; use uuid::Uuid; -generate_logging_api!("../../openapi/sled-agent.json"); +progenitor::generate_api!( + spec = "../../openapi/sled-agent.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + //TODO trade the manual transformations later in this file for the + // replace directives below? + replace = { + //Ipv4Network = ipnetwork::Ipv4Network, + SwitchLocation = omicron_common::api::external::SwitchLocation, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, + PortFec = omicron_common::api::internal::shared::PortFec, + PortSpeed = omicron_common::api::internal::shared::PortSpeed, + } +); impl omicron_common::api::external::ClientError for types::Error { fn message(&self) -> String { @@ -269,6 +291,12 @@ impl From for types::Ipv4Net { } } +impl From for types::Ipv4Network { + fn from(n: ipnetwork::Ipv4Network) -> Self { + Self::try_from(n.to_string()).unwrap_or_else(|e| panic!("{}: {}", n, e)) + } +} + impl From for types::Ipv6Net { fn from(n: ipnetwork::Ipv6Network) -> Self { Self::try_from(n.to_string()).unwrap_or_else(|e| panic!("{}: {}", n, e)) diff --git a/clients/wicketd-client/src/lib.rs b/clients/wicketd-client/src/lib.rs index ff45232520..982ec13780 100644 --- a/clients/wicketd-client/src/lib.rs +++ b/clients/wicketd-client/src/lib.rs @@ -42,8 +42,12 @@ progenitor::generate_api!( RackInitId = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, RackResetId = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, RackOperationStatus = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, - RackNetworkConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + RackNetworkConfigV1 = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, UplinkConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + PortConfigV1 = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + BgpPeerConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + BgpConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + RouteConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfigInsensitive = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfigSensitive = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, @@ -52,6 +56,8 @@ progenitor::generate_api!( replace = { Duration = std::time::Duration, Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, PutRssUserConfigInsensitive = wicket_common::rack_setup::PutRssUserConfigInsensitive, EventReportForWicketdEngineSpec = wicket_common::update_events::EventReport, StepEventForWicketdEngineSpec = wicket_common::update_events::StepEvent, diff --git a/common/src/address.rs b/common/src/address.rs index 0358787258..992e8f0406 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -39,6 +39,7 @@ pub const CLICKHOUSE_PORT: u16 = 8123; pub const CLICKHOUSE_KEEPER_PORT: u16 = 9181; pub const OXIMETER_PORT: u16 = 12223; pub const DENDRITE_PORT: u16 = 12224; +pub const MGD_PORT: u16 = 4676; pub const DDMD_PORT: u16 = 8000; pub const MGS_PORT: u16 = 12225; pub const WICKETD_PORT: u16 = 12226; @@ -47,6 +48,16 @@ pub const CRUCIBLE_PANTRY_PORT: u16 = 17000; pub const NEXUS_INTERNAL_PORT: u16 = 12221; +/// The port on which Nexus exposes its external API on the underlay network. +/// +/// This is used by the `wicketd` Nexus proxy to allow external API access via +/// the rack's tech port. +pub const NEXUS_TECHPORT_EXTERNAL_PORT: u16 = 12228; + +/// The port on which `wicketd` runs a Nexus external API proxy on the tech port +/// interface(s). +pub const WICKETD_NEXUS_PROXY_PORT: u16 = 12229; + pub const NTP_PORT: u16 = 123; // The number of ports available to an SNAT IP. @@ -162,6 +173,14 @@ impl Ipv6Subnet { } } +impl From for Ipv6Subnet { + fn from(net: Ipv6Network) -> Self { + // Ensure the address is set to within-prefix only components. + let net = Ipv6Network::new(net.network(), N).unwrap(); + Self { net: Ipv6Net(net) } + } +} + // We need a custom Deserialize to ensure that the subnet is what we expect. impl<'de, const N: u8> Deserialize<'de> for Ipv6Subnet { fn deserialize(deserializer: D) -> Result diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 53512408af..fcea57220d 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -12,6 +12,7 @@ pub mod http_pagination; use dropshot::HttpError; pub use error::*; +pub use crate::api::internal::shared::SwitchLocation; use anyhow::anyhow; use anyhow::Context; use api_identity::ObjectIdentity; @@ -98,6 +99,13 @@ pub struct DataPageParams<'a, NameType> { } impl<'a, NameType> DataPageParams<'a, NameType> { + pub fn max_page() -> Self { + Self { + marker: None, + direction: dropshot::PaginationOrder::Ascending, + limit: NonZeroU32::new(u32::MAX).unwrap(), + } + } /// Maps the marker type to a new type. /// /// Equivalent to [std::option::Option::map], because that's what it calls. @@ -400,7 +408,7 @@ impl SemverVersion { /// This is the official ECMAScript-compatible validation regex for /// semver: - /// https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + /// const VALIDATION_REGEX: &str = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"; } @@ -690,6 +698,8 @@ pub enum ResourceType { AddressLot, AddressLotBlock, BackgroundTask, + BgpConfig, + BgpAnnounceSet, Fleet, Silo, SiloUser, @@ -2459,9 +2469,6 @@ pub struct SwitchPortBgpPeerConfig { /// The port settings object this BGP configuration belongs to. pub port_settings_id: Uuid, - /// The id for the set of prefixes announced in this peer configuration. - pub bgp_announce_set_id: Uuid, - /// The id of the global BGP configuration referenced by this peer /// configuration. pub bgp_config_id: Uuid, @@ -2476,7 +2483,9 @@ pub struct SwitchPortBgpPeerConfig { } /// A base BGP configuration. -#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +#[derive( + ObjectIdentity, Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, +)] pub struct BgpConfig { #[serde(flatten)] pub identity: IdentityMetadata, @@ -2528,6 +2537,72 @@ pub struct SwitchPortAddressConfig { pub interface_name: String, } +/// The current state of a BGP peer. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum BgpPeerState { + /// Initial state. Refuse all incomming BGP connections. No resources + /// allocated to peer. + Idle, + + /// Waiting for the TCP connection to be completed. + Connect, + + /// Trying to acquire peer by listening for and accepting a TCP connection. + Active, + + /// Waiting for open message from peer. + OpenSent, + + /// Waiting for keepaliave or notification from peer. + OpenConfirm, + + /// Synchronizing with peer. + SessionSetup, + + /// Session established. Able to exchange update, notification and keepliave + /// messages with peers. + Established, +} + +/// The current status of a BGP peer. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct BgpPeerStatus { + /// IP address of the peer. + pub addr: IpAddr, + + /// Local autonomous system number. + pub local_asn: u32, + + /// Remote autonomous system number. + pub remote_asn: u32, + + /// State of the peer. + pub state: BgpPeerState, + + /// Time of last state change. + pub state_duration_millis: u64, + + /// Switch with the peer session. + pub switch: SwitchLocation, +} + +/// A route imported from a BGP peer. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct BgpImportedRouteIpv4 { + /// The destination network prefix. + pub prefix: Ipv4Net, + + /// The nexthop the prefix is reachable through. + pub nexthop: Ipv4Addr, + + /// BGP identifier of the originating router. + pub id: u32, + + /// Switch the route is imported into. + pub switch: SwitchLocation, +} + #[cfg(test)] mod test { use serde::Deserialize; diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 9e3f3ec1f6..784da8fcc6 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -5,7 +5,7 @@ //! Types shared between Nexus and Sled Agent. use crate::api::external::{self, Name}; -use ipnetwork::Ipv4Network; +use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{ @@ -68,18 +68,88 @@ pub struct SourceNatConfig { pub last_port: u16, } +// We alias [`RackNetworkConfig`] to the current version of the protocol, so +// that we can convert between versions as necessary. +pub type RackNetworkConfig = RackNetworkConfigV1; + /// Initial network configuration #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] -pub struct RackNetworkConfig { +pub struct RackNetworkConfigV1 { + pub rack_subnet: Ipv6Network, // TODO: #3591 Consider making infra-ip ranges implicit for uplinks /// First ip address to be used for configuring network infrastructure pub infra_ip_first: Ipv4Addr, /// Last ip address to be used for configuring network infrastructure pub infra_ip_last: Ipv4Addr, /// Uplinks for connecting the rack to external networks - pub uplinks: Vec, + pub ports: Vec, + /// BGP configurations for connecting the rack to external networks + pub bgp: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct BgpConfig { + /// The autonomous system number for the BGP configuration. + pub asn: u32, + /// The set of prefixes for the BGP router to originate. + pub originate: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct BgpPeerConfig { + /// The autonomous sysetm number of the router the peer belongs to. + pub asn: u32, + /// Switch port the peer is reachable on. + pub port: String, + /// Address of the peer. + pub addr: Ipv4Addr, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct RouteConfig { + /// The destination of the route. + pub destination: IpNetwork, + /// The nexthop/gateway address. + pub nexthop: IpAddr, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct PortConfigV1 { + /// The set of routes associated with this port. + pub routes: Vec, + /// This port's addresses. + pub addresses: Vec, + /// Switch the port belongs to. + pub switch: SwitchLocation, + /// Nmae of the port this config applies to. + pub port: String, + /// Port speed. + pub uplink_port_speed: PortSpeed, + /// Port forward error correction type. + pub uplink_port_fec: PortFec, + /// BGP peers on this port + pub bgp_peers: Vec, } +impl From for PortConfigV1 { + fn from(value: UplinkConfig) -> Self { + PortConfigV1 { + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: value.gateway_ip.into(), + }], + addresses: vec![value.uplink_cidr.into()], + switch: value.switch, + port: value.uplink_port, + uplink_port_speed: value.uplink_port_speed, + uplink_port_fec: value.uplink_port_fec, + bgp_peers: vec![], + } + } +} + +/// Deprecated, use PortConfigV1 instead. Cannot actually deprecate due to +/// #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct UplinkConfig { /// Gateway address @@ -99,9 +169,41 @@ pub struct UplinkConfig { pub uplink_vid: Option, } +/// A set of switch uplinks. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct SwitchPorts { + pub uplinks: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct HostPortConfig { + /// Switchport to use for external connectivity + pub port: String, + + /// IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport + /// (must be in infra_ip pool) + pub addrs: Vec, +} + +impl From for HostPortConfig { + fn from(x: PortConfigV1) -> Self { + Self { port: x.port, addrs: x.addresses } + } +} + /// Identifies switch physical location #[derive( - Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema, Hash, Eq, + Clone, + Copy, + Debug, + Deserialize, + Serialize, + PartialEq, + JsonSchema, + Hash, + Eq, + PartialOrd, + Ord, )] #[serde(rename_all = "snake_case")] pub enum SwitchLocation { diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index 9be58d3222..4e821e2676 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -5,6 +5,7 @@ //! Configuration parameters to Nexus that are usually only known //! at deployment time. +use crate::address::NEXUS_TECHPORT_EXTERNAL_PORT; use crate::api::internal::shared::SwitchLocation; use super::address::{Ipv6Subnet, RACK_PREFIX}; @@ -132,6 +133,19 @@ pub struct DeploymentConfig { pub id: Uuid, /// Uuid of the Rack where Nexus is executing. pub rack_id: Uuid, + /// Port on which the "techport external" dropshot server should listen. + /// This dropshot server copies _most_ of its config from + /// `dropshot_external` (so that it matches TLS, etc.), but builds its + /// listening address by combining `dropshot_internal`'s IP address with + /// this port. + /// + /// We use `serde(default = ...)` to ensure we don't break any serialized + /// configs that were created before this field was added. In production we + /// always expect this port to be constant, but we need to be able to + /// override it when running tests. + #[schemars(skip)] + #[serde(default = "default_techport_external_server_port")] + pub techport_external_server_port: u16, /// Dropshot configuration for the external API server. #[schemars(skip)] // TODO we're protected against dropshot changes pub dropshot_external: ConfigDropshotWithTls, @@ -147,6 +161,10 @@ pub struct DeploymentConfig { pub external_dns_servers: Vec, } +fn default_techport_external_server_port() -> u16 { + NEXUS_TECHPORT_EXTERNAL_PORT +} + impl DeploymentConfig { /// Load a `DeploymentConfig` from the given TOML file /// @@ -223,6 +241,12 @@ pub struct DpdConfig { pub address: SocketAddr, } +/// Configuration for the `Dendrite` dataplane daemon. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct MgdConfig { + pub address: SocketAddr, +} + // A deserializable type that does no validation on the tunable parameters. #[derive(Clone, Debug, Deserialize, PartialEq)] struct UnvalidatedTunables { @@ -396,6 +420,9 @@ pub struct PackageConfig { /// `Dendrite` dataplane daemon configuration #[serde(default)] pub dendrite: HashMap, + /// Maghemite mgd daemon configuration + #[serde(default)] + pub mgd: HashMap, /// Background task configuration pub background_tasks: BackgroundTaskConfig, /// Default Crucible region allocation strategy @@ -467,11 +494,12 @@ impl std::fmt::Display for SchemeName { #[cfg(test)] mod test { use super::{ - AuthnConfig, BackgroundTaskConfig, Config, ConfigDropshotWithTls, - ConsoleConfig, Database, DeploymentConfig, DnsTasksConfig, DpdConfig, + default_techport_external_server_port, AuthnConfig, + BackgroundTaskConfig, Config, ConfigDropshotWithTls, ConsoleConfig, + Database, DeploymentConfig, DnsTasksConfig, DpdConfig, ExternalEndpointsConfig, InternalDns, InventoryConfig, LoadError, - LoadErrorKind, PackageConfig, SchemeName, TimeseriesDbConfig, Tunables, - UpdatesConfig, + LoadErrorKind, MgdConfig, PackageConfig, SchemeName, + TimeseriesDbConfig, Tunables, UpdatesConfig, }; use crate::address::{Ipv6Subnet, RACK_PREFIX}; use crate::api::internal::shared::SwitchLocation; @@ -609,6 +637,8 @@ mod test { type = "from_dns" [dendrite.switch0] address = "[::1]:12224" + [mgd.switch0] + address = "[::1]:4676" [background_tasks] dns_internal.period_secs_config = 1 dns_internal.period_secs_servers = 2 @@ -637,6 +667,8 @@ mod test { rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f" .parse() .unwrap(), + techport_external_server_port: + default_techport_external_server_port(), dropshot_external: ConfigDropshotWithTls { tls: false, dropshot: ConfigDropshot { @@ -691,6 +723,13 @@ mod test { .unwrap(), } )]), + mgd: HashMap::from([( + SwitchLocation::Switch0, + MgdConfig { + address: SocketAddr::from_str("[::1]:4676") + .unwrap(), + } + )]), background_tasks: BackgroundTaskConfig { dns_internal: DnsTasksConfig { period_secs_config: Duration::from_secs(1), @@ -740,6 +779,7 @@ mod test { [deployment] id = "28b90dc4-c22a-65ba-f49a-f051fe01208f" rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f" + techport_external_server_port = 12345 external_dns_servers = [ "1.1.1.1", "9.9.9.9" ] [deployment.dropshot_external] bind_address = "10.1.2.3:4567" @@ -777,6 +817,7 @@ mod test { config.pkg.authn.schemes_external, vec![SchemeName::Spoof, SchemeName::SessionCookie], ); + assert_eq!(config.deployment.techport_external_server_port, 12345); } #[test] diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 54e344a04d..efcefdea43 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -43,7 +43,10 @@ use nexus_db_model::Instance; use nexus_db_model::InvCollection; use nexus_db_model::Project; use nexus_db_model::Region; +use nexus_db_model::RegionSnapshot; use nexus_db_model::Sled; +use nexus_db_model::Snapshot; +use nexus_db_model::SnapshotState; use nexus_db_model::SwCaboose; use nexus_db_model::Vmm; use nexus_db_model::Zpool; @@ -67,6 +70,7 @@ use omicron_common::postgres_config::PostgresConfigWithUrl; use std::cmp::Ordering; use std::collections::BTreeMap; use std::collections::BTreeSet; +use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Display; use std::num::NonZeroU32; @@ -137,16 +141,18 @@ enum DbCommands { Disks(DiskArgs), /// Print information about internal and external DNS Dns(DnsArgs), - /// Print information about customer instances - Instances, /// Print information about collected hardware/software inventory Inventory(InventoryArgs), - /// Print information about the network - Network(NetworkArgs), /// Print information about control plane services Services(ServicesArgs), /// Print information about sleds Sleds, + /// Print information about customer instances + Instances, + /// Print information about the network + Network(NetworkArgs), + /// Print information about snapshots + Snapshots(SnapshotArgs), } #[derive(Debug, Args)] @@ -283,6 +289,26 @@ enum NetworkCommands { ListEips, } +#[derive(Debug, Args)] +struct SnapshotArgs { + #[command(subcommand)] + command: SnapshotCommands, +} + +#[derive(Debug, Subcommand)] +enum SnapshotCommands { + /// Get info for a specific snapshot + Info(SnapshotInfoArgs), + /// Summarize current snapshots + List, +} + +#[derive(Debug, Args)] +struct SnapshotInfoArgs { + /// The UUID of the snapshot + uuid: Uuid, +} + impl DbArgs { /// Run a `omdb db` subcommand. pub(crate) async fn run_cmd( @@ -356,20 +382,10 @@ impl DbArgs { cmd_db_dns_names(&opctx, &datastore, self.fetch_limit, args) .await } - DbCommands::Instances => { - cmd_db_instances(&datastore, self.fetch_limit).await - } DbCommands::Inventory(inventory_args) => { cmd_db_inventory(&datastore, self.fetch_limit, inventory_args) .await } - DbCommands::Network(NetworkArgs { - command: NetworkCommands::ListEips, - verbose, - }) => { - cmd_db_eips(&opctx, &datastore, self.fetch_limit, *verbose) - .await - } DbCommands::Services(ServicesArgs { command: ServicesCommands::ListInstances, }) => { @@ -393,6 +409,22 @@ impl DbArgs { DbCommands::Sleds => { cmd_db_sleds(&opctx, &datastore, self.fetch_limit).await } + DbCommands::Instances => { + cmd_db_instances(&opctx, &datastore, self.fetch_limit).await + } + DbCommands::Network(NetworkArgs { + command: NetworkCommands::ListEips, + verbose, + }) => { + cmd_db_eips(&opctx, &datastore, self.fetch_limit, *verbose) + .await + } + DbCommands::Snapshots(SnapshotArgs { + command: SnapshotCommands::Info(uuid), + }) => cmd_db_snapshot_info(&opctx, &datastore, uuid).await, + DbCommands::Snapshots(SnapshotArgs { + command: SnapshotCommands::List, + }) => cmd_db_snapshot_list(&datastore, self.fetch_limit).await, } } } @@ -539,6 +571,8 @@ async fn cmd_db_disk_info( disk_name: String, instance_name: String, propolis_zone: String, + volume_id: String, + disk_state: String, } // The rows describing the downstairs regions for this disk/volume @@ -572,7 +606,7 @@ async fn cmd_db_disk_info( // If the disk is attached to an instance, show information // about that instance. - if let Some(instance_uuid) = disk.runtime().attach_instance_id { + let usr = if let Some(instance_uuid) = disk.runtime().attach_instance_id { // Get the instance this disk is attached to use db::schema::instance::dsl as instance_dsl; use db::schema::vmm::dsl as vmm_dsl; @@ -599,7 +633,7 @@ async fn cmd_db_disk_info( let instance_name = instance.instance().name().to_string(); let disk_name = disk.name().to_string(); - let usr = if instance.vmm().is_some() { + if instance.vmm().is_some() { let propolis_id = instance.instance().runtime().propolis_id.unwrap(); let my_sled_id = instance.sled_id().unwrap(); @@ -615,27 +649,32 @@ async fn cmd_db_disk_info( disk_name, instance_name, propolis_zone: format!("oxz_propolis-server_{}", propolis_id), + volume_id: disk.volume_id.to_string(), + disk_state: disk.runtime_state.disk_state.to_string(), } } else { UpstairsRow { host_serial: NOT_ON_SLED_MSG.to_string(), - propolis_zone: NO_ACTIVE_PROPOLIS_MSG.to_string(), disk_name, instance_name, + propolis_zone: NO_ACTIVE_PROPOLIS_MSG.to_string(), + volume_id: disk.volume_id.to_string(), + disk_state: disk.runtime_state.disk_state.to_string(), } - }; - rows.push(usr); + } } else { // If the disk is not attached to anything, just print empty // fields. - let usr = UpstairsRow { + UpstairsRow { host_serial: "-".to_string(), disk_name: disk.name().to_string(), instance_name: "-".to_string(), propolis_zone: "-".to_string(), - }; - rows.push(usr); - } + volume_id: disk.volume_id.to_string(), + disk_state: disk.runtime_state.disk_state.to_string(), + } + }; + rows.push(usr); let table = tabled::Table::new(rows) .with(tabled::settings::Style::empty()) @@ -689,19 +728,25 @@ async fn cmd_db_disk_physical( limit: NonZeroU32, args: &DiskPhysicalArgs, ) -> Result<(), anyhow::Error> { + let conn = datastore.pool_connection_for_tests().await?; + // We start by finding any zpools that are using the physical disk. use db::schema::zpool::dsl as zpool_dsl; let zpools = zpool_dsl::zpool .filter(zpool_dsl::time_deleted.is_null()) .filter(zpool_dsl::physical_disk_id.eq(args.uuid)) .select(Zpool::as_select()) - .load_async(&*datastore.pool_connection_for_tests().await?) + .load_async(&*conn) .await .context("loading zpool from pysical disk id")?; let mut sled_ids = HashSet::new(); let mut dataset_ids = HashSet::new(); + if zpools.is_empty() { + println!("Found no zpools on physical disk UUID {}", args.uuid); + return Ok(()); + } // The current plan is a single zpool per physical disk, so we expect that // this will have a single item. However, If single zpool per disk ever // changes, this code will still work. @@ -715,7 +760,7 @@ async fn cmd_db_disk_physical( .filter(dataset_dsl::time_deleted.is_null()) .filter(dataset_dsl::pool_id.eq(zp.id())) .select(Dataset::as_select()) - .load_async(&*datastore.pool_connection_for_tests().await?) + .load_async(&*conn) .await .context("loading dataset")?; @@ -740,6 +785,7 @@ async fn cmd_db_disk_physical( my_sled.serial_number() ); } + println!("DATASETS: {:?}", dataset_ids); let mut volume_ids = HashSet::new(); // Now, take the list of datasets we found and search all the regions @@ -750,7 +796,7 @@ async fn cmd_db_disk_physical( let regions = region_dsl::region .filter(region_dsl::dataset_id.eq(did)) .select(Region::as_select()) - .load_async(&*datastore.pool_connection_for_tests().await?) + .load_async(&*conn) .await .context("loading region")?; @@ -760,7 +806,7 @@ async fn cmd_db_disk_physical( } // At this point, we have a list of volume IDs that contain a region - // that is part of a dataset on a pool on our disk. The final step is + // that is part of a dataset on a pool on our disk. The next step is // to find the virtual disks associated with these volume IDs and // display information about those disks. use db::schema::disk::dsl; @@ -769,7 +815,7 @@ async fn cmd_db_disk_physical( .filter(dsl::volume_id.eq_any(volume_ids)) .limit(i64::from(u32::from(limit))) .select(Disk::as_select()) - .load_async(&*datastore.pool_connection_for_tests().await?) + .load_async(&*conn) .await .context("loading disks")?; @@ -778,7 +824,7 @@ async fn cmd_db_disk_physical( #[derive(Tabled)] #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] struct DiskRow { - name: String, + disk_name: String, id: String, state: String, instance_name: String, @@ -797,7 +843,7 @@ async fn cmd_db_disk_physical( .filter(instance_dsl::id.eq(instance_uuid)) .limit(1) .select(Instance::as_select()) - .load_async(&*datastore.pool_connection_for_tests().await?) + .load_async(&*conn) .await .context("loading requested instance")?; @@ -811,7 +857,7 @@ async fn cmd_db_disk_physical( }; rows.push(DiskRow { - name: disk.name().to_string(), + disk_name: disk.name().to_string(), id: disk.id().to_string(), state: disk.runtime().disk_state, instance_name, @@ -823,6 +869,75 @@ async fn cmd_db_disk_physical( .with(tabled::settings::Padding::new(0, 1, 0, 0)) .to_string(); + println!("{}", table); + + // Collect the region_snapshots associated with the dataset IDs + use db::schema::region_snapshot::dsl as region_snapshot_dsl; + let region_snapshots = region_snapshot_dsl::region_snapshot + .filter(region_snapshot_dsl::dataset_id.eq_any(dataset_ids)) + .limit(i64::from(u32::from(limit))) + .select(RegionSnapshot::as_select()) + .load_async(&*conn) + .await + .context("loading region snapshots")?; + + check_limit(®ion_snapshots, limit, || { + "listing region snapshots".to_string() + }); + + // The row describing the region_snapshot. + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct RegionSnapshotRow { + dataset_id: String, + region_id: String, + snapshot_id: String, + volume_references: String, + } + let mut rsnap = Vec::new(); + + // From each region snapshot: + // Collect the snapshot IDs for later use. + // Display the region snapshot rows. + let mut snapshot_ids = HashSet::new(); + for rs in region_snapshots { + snapshot_ids.insert(rs.snapshot_id); + let rs = RegionSnapshotRow { + dataset_id: rs.dataset_id.to_string(), + region_id: rs.region_id.to_string(), + snapshot_id: rs.snapshot_id.to_string(), + volume_references: rs.volume_references.to_string(), + }; + rsnap.push(rs); + } + let table = tabled::Table::new(rsnap) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + + println!("{}", table); + + // Get the snapshots from the list of IDs we built above. + // Display information about those snapshots. + use db::schema::snapshot::dsl as snapshot_dsl; + let snapshots = snapshot_dsl::snapshot + .filter(snapshot_dsl::time_deleted.is_null()) + .filter(snapshot_dsl::id.eq_any(snapshot_ids)) + .limit(i64::from(u32::from(limit))) + .select(Snapshot::as_select()) + .load_async(&*conn) + .await + .context("loading snapshots")?; + + check_limit(&snapshots, limit, || "listing snapshots".to_string()); + + let rows = + snapshots.into_iter().map(|snapshot| SnapshotRow::from(snapshot)); + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + println!("{}", table); Ok(()) } @@ -839,6 +954,154 @@ struct ServiceInstanceRow { sled_serial: String, } +// Snapshots +fn format_snapshot(state: &SnapshotState) -> impl Display { + match state { + SnapshotState::Creating => "creating".to_string(), + SnapshotState::Ready => "ready".to_string(), + SnapshotState::Faulted => "faulted".to_string(), + SnapshotState::Destroyed => "destroyed".to_string(), + } +} + +// The row describing the snapshot +#[derive(Tabled)] +#[tabled(rename_all = "SCREAMING_SNAKE_CASE")] +struct SnapshotRow { + snap_name: String, + id: String, + state: String, + size: String, + source_disk_id: String, + source_volume_id: String, + destination_volume_id: String, +} + +impl From for SnapshotRow { + fn from(s: Snapshot) -> Self { + SnapshotRow { + snap_name: s.name().to_string(), + id: s.id().to_string(), + state: format_snapshot(&s.state).to_string(), + size: s.size.to_string(), + source_disk_id: s.disk_id.to_string(), + source_volume_id: s.volume_id.to_string(), + destination_volume_id: s.destination_volume_id.to_string(), + } + } +} + +/// Run `omdb db snapshot list`. +async fn cmd_db_snapshot_list( + datastore: &DataStore, + limit: NonZeroU32, +) -> Result<(), anyhow::Error> { + let ctx = || "listing snapshots".to_string(); + + use db::schema::snapshot::dsl; + let snapshots = dsl::snapshot + .filter(dsl::time_deleted.is_null()) + .limit(i64::from(u32::from(limit))) + .select(Snapshot::as_select()) + .load_async(&*datastore.pool_connection_for_tests().await?) + .await + .context("loading snapshots")?; + + check_limit(&snapshots, limit, ctx); + + let rows = + snapshots.into_iter().map(|snapshot| SnapshotRow::from(snapshot)); + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + + println!("{}", table); + + Ok(()) +} + +/// Run `omdb db snapshot info `. +async fn cmd_db_snapshot_info( + opctx: &OpContext, + datastore: &DataStore, + args: &SnapshotInfoArgs, +) -> Result<(), anyhow::Error> { + // The rows describing the downstairs regions for this snapshot/volume + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct DownstairsRow { + host_serial: String, + region: String, + zone: String, + physical_disk: String, + } + + use db::schema::snapshot::dsl as snapshot_dsl; + let snapshots = snapshot_dsl::snapshot + .filter(snapshot_dsl::id.eq(args.uuid)) + .limit(1) + .select(Snapshot::as_select()) + .load_async(&*datastore.pool_connection_for_tests().await?) + .await + .context("loading requested snapshot")?; + + let mut dest_volume_ids = Vec::new(); + let rows = snapshots.into_iter().map(|snapshot| { + dest_volume_ids.push(snapshot.destination_volume_id); + SnapshotRow::from(snapshot) + }); + if rows.len() == 0 { + bail!("No snapshout with UUID: {} found", args.uuid); + } + + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + + println!("{}", table); + + for vol_id in dest_volume_ids { + // Get the dataset backing this volume. + let regions = datastore.get_allocated_regions(vol_id).await?; + + let mut rows = Vec::with_capacity(3); + for (dataset, region) in regions { + let my_pool_id = dataset.pool_id; + let (_, my_zpool) = LookupPath::new(opctx, datastore) + .zpool_id(my_pool_id) + .fetch() + .await + .context("failed to look up zpool")?; + + let my_sled_id = my_zpool.sled_id; + + let (_, my_sled) = LookupPath::new(opctx, datastore) + .sled_id(my_sled_id) + .fetch() + .await + .context("failed to look up sled")?; + + rows.push(DownstairsRow { + host_serial: my_sled.serial_number().to_string(), + region: region.id().to_string(), + zone: format!("oxz_crucible_{}", dataset.id()), + physical_disk: my_zpool.physical_disk_id.to_string(), + }); + } + + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + + println!("{}", table); + } + + Ok(()) +} + /// Run `omdb db services list-instances`. async fn cmd_db_services_list_instances( opctx: &OpContext, @@ -1006,25 +1269,17 @@ async fn cmd_db_sleds( #[derive(Tabled)] #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] struct CustomerInstanceRow { - id: Uuid, + id: String, + name: String, state: String, propolis_id: MaybePropolisId, sled_id: MaybeSledId, -} - -impl From for CustomerInstanceRow { - fn from(i: InstanceAndActiveVmm) -> Self { - CustomerInstanceRow { - id: i.instance().id(), - state: format!("{:?}", i.effective_state()), - propolis_id: (&i).into(), - sled_id: (&i).into(), - } - } + host_serial: String, } /// Run `omdb db instances`: list data about customer VMs. async fn cmd_db_instances( + opctx: &OpContext, datastore: &DataStore, limit: NonZeroU32, ) -> Result<(), anyhow::Error> { @@ -1049,7 +1304,42 @@ async fn cmd_db_instances( let ctx = || "listing instances".to_string(); check_limit(&instances, limit, ctx); - let rows = instances.into_iter().map(|i| CustomerInstanceRow::from(i)); + let mut rows = Vec::new(); + let mut h_to_s: HashMap = HashMap::new(); + + for i in instances { + let host_serial = if i.vmm().is_some() { + if let std::collections::hash_map::Entry::Vacant(e) = + h_to_s.entry(i.sled_id().unwrap()) + { + let (_, my_sled) = LookupPath::new(opctx, datastore) + .sled_id(i.sled_id().unwrap()) + .fetch() + .await + .context("failed to look up sled")?; + + let host_serial = my_sled.serial_number().to_string(); + e.insert(host_serial.to_string()); + host_serial.to_string() + } else { + h_to_s.get(&i.sled_id().unwrap()).unwrap().to_string() + } + } else { + "-".to_string() + }; + + let cir = CustomerInstanceRow { + id: i.instance().id().to_string(), + name: i.instance().name().to_string(), + state: i.effective_state().to_string(), + propolis_id: (&i).into(), + sled_id: (&i).into(), + host_serial, + }; + + rows.push(cir); + } + let table = tabled::Table::new(rows) .with(tabled::settings::Style::empty()) .with(tabled::settings::Padding::new(0, 1, 0, 0)) diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 2cf42c1b22..7949c1eb61 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -2,12 +2,12 @@ EXECUTING COMMAND: omdb ["db", "--db-url", "postgresql://root@[::1]:REDACTED_POR termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected () ============================================= EXECUTING COMMAND: omdb ["db", "--db-url", "junk", "sleds"] termination: Exited(2) @@ -177,25 +177,25 @@ EXECUTING COMMAND: omdb ["db", "sleds"] termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected () ============================================= EXECUTING COMMAND: omdb ["--dns-server", "[::1]:REDACTED_PORT", "db", "sleds"] termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected () ============================================= diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 2d03c697f5..8162b6d9de 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -1,3 +1,13 @@ +EXECUTING COMMAND: omdb ["db", "disks", "list"] +termination: Exited(0) +--------------------------------------------- +stdout: +NAME ID SIZE STATE ATTACHED_TO +--------------------------------------------- +stderr: +note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable +note: database schema version matches expected () +============================================= EXECUTING COMMAND: omdb ["db", "dns", "show"] termination: Exited(0) --------------------------------------------- @@ -8,7 +18,7 @@ external oxide-dev.test 2 create silo: "tes --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected () ============================================= EXECUTING COMMAND: omdb ["db", "dns", "diff", "external", "2"] termination: Exited(0) @@ -24,7 +34,7 @@ changes: names added: 1, names removed: 0 --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected () ============================================= EXECUTING COMMAND: omdb ["db", "dns", "names", "external", "2"] termination: Exited(0) @@ -36,7 +46,17 @@ External zone: oxide-dev.test --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected () +============================================= +EXECUTING COMMAND: omdb ["db", "instances"] +termination: Exited(0) +--------------------------------------------- +stdout: +ID NAME STATE PROPOLIS_ID SLED_ID HOST_SERIAL +--------------------------------------------- +stderr: +note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable +note: database schema version matches expected () ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-instances"] termination: Exited(0) @@ -49,10 +69,12 @@ Dendrite REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT ExternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 InternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 Nexus REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_PORT sim-b6d65341 +Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 +Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected () ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-by-sled"] termination: Exited(0) @@ -67,22 +89,24 @@ sled: sim-b6d65341 (id REDACTED_UUID_REDACTED_UUID_REDACTED) ExternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT InternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT Nexus REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_PORT + Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT + Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected () ============================================= EXECUTING COMMAND: omdb ["db", "sleds"] termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected () ============================================= EXECUTING COMMAND: omdb ["mgs", "inventory"] termination: Exited(0) diff --git a/dev-tools/omdb/tests/test_all_output.rs b/dev-tools/omdb/tests/test_all_output.rs index 90e93ee429..dc681712eb 100644 --- a/dev-tools/omdb/tests/test_all_output.rs +++ b/dev-tools/omdb/tests/test_all_output.rs @@ -37,10 +37,12 @@ async fn test_omdb_usage_errors() { // Command help output &["db"], &["db", "--help"], + &["db", "disks"], &["db", "dns"], &["db", "dns", "diff"], &["db", "dns", "names"], &["db", "services"], + &["db", "snapshots"], &["db", "network"], &["mgs"], &["nexus"], @@ -71,9 +73,11 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) { let mgs_url = format!("http://{}/", gwtestctx.client.bind_address); let mut output = String::new(); let invocations: &[&[&'static str]] = &[ + &["db", "disks", "list"], &["db", "dns", "show"], &["db", "dns", "diff", "external", "2"], &["db", "dns", "names", "external", "2"], + &["db", "instances"], &["db", "services", "list-instances"], &["db", "services", "list-by-sled"], &["db", "sleds"], @@ -304,5 +308,16 @@ fn redact_variable(input: &str) -> String { .replace_all(&s, "ms") .to_string(); + let s = regex::Regex::new( + r"note: database schema version matches expected \(\d+\.\d+\.\d+\)", + ) + .unwrap() + .replace_all( + &s, + "note: database schema version matches expected \ + ()", + ) + .to_string(); + s } diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index dc75278fc3..e859c325a5 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -92,11 +92,12 @@ Usage: omdb db [OPTIONS] Commands: disks Print information about disks dns Print information about internal and external DNS - instances Print information about customer instances inventory Print information about collected hardware/software inventory - network Print information about the network services Print information about control plane services sleds Print information about sleds + instances Print information about customer instances + network Print information about the network + snapshots Print information about snapshots help Print this message or the help of the given subcommand(s) Options: @@ -115,11 +116,12 @@ Usage: omdb db [OPTIONS] Commands: disks Print information about disks dns Print information about internal and external DNS - instances Print information about customer instances inventory Print information about collected hardware/software inventory - network Print information about the network services Print information about control plane services sleds Print information about sleds + instances Print information about customer instances + network Print information about the network + snapshots Print information about snapshots help Print this message or the help of the given subcommand(s) Options: @@ -129,6 +131,25 @@ Options: --------------------------------------------- stderr: ============================================= +EXECUTING COMMAND: omdb ["db", "disks"] +termination: Exited(2) +--------------------------------------------- +stdout: +--------------------------------------------- +stderr: +Print information about disks + +Usage: omdb db disks + +Commands: + info Get info for a specific disk + list Summarize current disks + physical Determine what crucible resources are on the given physical disk + help Print this message or the help of the given subcommand(s) + +Options: + -h, --help Print help +============================================= EXECUTING COMMAND: omdb ["db", "dns"] termination: Exited(2) --------------------------------------------- @@ -191,6 +212,24 @@ Commands: list-by-sled List service instances, grouped by sled help Print this message or the help of the given subcommand(s) +Options: + -h, --help Print help +============================================= +EXECUTING COMMAND: omdb ["db", "snapshots"] +termination: Exited(2) +--------------------------------------------- +stdout: +--------------------------------------------- +stderr: +Print information about snapshots + +Usage: omdb db snapshots + +Commands: + info Get info for a specific snapshot + list Summarize current snapshots + help Print this message or the help of the given subcommand(s) + Options: -h, --help Print help ============================================= diff --git a/env.sh b/env.sh index 5b1e2b34ac..483a89f597 100644 --- a/env.sh +++ b/env.sh @@ -9,5 +9,6 @@ OMICRON_WS="$(cd $(dirname "${BASH_SOURCE[0]}") && echo $PWD)" export PATH="$OMICRON_WS/out/cockroachdb/bin:$PATH" export PATH="$OMICRON_WS/out/clickhouse:$PATH" export PATH="$OMICRON_WS/out/dendrite-stub/bin:$PATH" +export PATH="$OMICRON_WS/out/mgd/root/opt/oxide/mgd/bin:$PATH" unset OMICRON_WS set +o xtrace diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 07934a6ad3..9cf41f6c2e 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -7,9 +7,9 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true async-trait.workspace = true +base64.workspace = true ciborium.workspace = true clap.workspace = true -crucible-smf.workspace = true dropshot.workspace = true futures.workspace = true gateway-messages.workspace = true @@ -17,6 +17,7 @@ gateway-sp-comms.workspace = true hex.workspace = true http.workspace = true hyper.workspace = true +illumos-utils.workspace = true ipcc-key-value.workspace = true omicron-common.workspace = true once_cell.workspace = true diff --git a/gateway/src/bin/mgs.rs b/gateway/src/bin/mgs.rs index cb9070a9a5..81b10ef669 100644 --- a/gateway/src/bin/mgs.rs +++ b/gateway/src/bin/mgs.rs @@ -85,12 +85,11 @@ async fn do_run() -> Result<(), CmdError> { )) })?; - let mut signals = - Signals::new(&[signal::SIGUSR1]).map_err(|e| { - CmdError::Failure(format!( - "failed to set up signal handler: {e}" - )) - })?; + let mut signals = Signals::new([signal::SIGUSR1]).map_err(|e| { + CmdError::Failure(format!( + "failed to set up signal handler: {e}" + )) + })?; let (id, addresses, rack_id) = if id_and_address_from_smf { let config = read_smf_config()?; @@ -141,7 +140,11 @@ async fn do_run() -> Result<(), CmdError> { #[cfg(target_os = "illumos")] fn read_smf_config() -> Result { - use crucible_smf::{Scf, ScfError}; + fn scf_to_cmd_err(err: illumos_utils::scf::ScfError) -> CmdError { + CmdError::Failure(err.to_string()) + } + + use illumos_utils::scf::ScfHandle; // Name of our config property group; must match our SMF manifest.xml. const CONFIG_PG: &str = "config"; @@ -155,107 +158,46 @@ fn read_smf_config() -> Result { // Name of the property within CONFIG_PG for our rack ID. const PROP_RACK_ID: &str = "rack_id"; - // This function is pretty boilerplate-y; we can reduce it by using this - // error type to help us construct a `CmdError::Failure(_)` string. It - // assumes (for the purposes of error messages) any property being fetched - // lives under the `CONFIG_PG` property group. - #[derive(Debug, thiserror::Error)] - enum Error { - #[error("failed to create scf handle: {0}")] - ScfHandle(ScfError), - #[error("failed to get self smf instance: {0}")] - SelfInstance(ScfError), - #[error("failed to get self running snapshot: {0}")] - RunningSnapshot(ScfError), - #[error("failed to get propertygroup `{CONFIG_PG}`: {0}")] - GetPg(ScfError), - #[error("missing propertygroup `{CONFIG_PG}`")] - MissingPg, - #[error("failed to get property `{CONFIG_PG}/{prop}`: {err}")] - GetProperty { prop: &'static str, err: ScfError }, - #[error("missing property `{CONFIG_PG}/{prop}`")] - MissingProperty { prop: &'static str }, - #[error("failed to get value for `{CONFIG_PG}/{prop}`: {err}")] - GetValue { prop: &'static str, err: ScfError }, - #[error("failed to get values for `{CONFIG_PG}/{prop}`: {err}")] - GetValues { prop: &'static str, err: ScfError }, - #[error("failed to get value for `{CONFIG_PG}/{prop}`")] - MissingValue { prop: &'static str }, - #[error("failed to get `{CONFIG_PG}/{prop} as a string: {err}")] - ValueAsString { prop: &'static str, err: ScfError }, - } - - impl From for CmdError { - fn from(err: Error) -> Self { - Self::Failure(err.to_string()) - } - } - - let scf = Scf::new().map_err(Error::ScfHandle)?; - let instance = scf.get_self_instance().map_err(Error::SelfInstance)?; - let snapshot = - instance.get_running_snapshot().map_err(Error::RunningSnapshot)?; - - let config = snapshot - .get_pg("config") - .map_err(Error::GetPg)? - .ok_or(Error::MissingPg)?; + let scf = ScfHandle::new().map_err(scf_to_cmd_err)?; + let instance = scf.self_instance().map_err(scf_to_cmd_err)?; + let snapshot = instance.running_snapshot().map_err(scf_to_cmd_err)?; + let config = snapshot.property_group(CONFIG_PG).map_err(scf_to_cmd_err)?; - let prop_id = config - .get_property(PROP_ID) - .map_err(|err| Error::GetProperty { prop: PROP_ID, err })? - .ok_or_else(|| Error::MissingProperty { prop: PROP_ID })? - .value() - .map_err(|err| Error::GetValue { prop: PROP_ID, err })? - .ok_or(Error::MissingValue { prop: PROP_ID })? - .as_string() - .map_err(|err| Error::ValueAsString { prop: PROP_ID, err })?; + let prop_id = config.value_as_string(PROP_ID).map_err(scf_to_cmd_err)?; let prop_id = Uuid::try_parse(&prop_id).map_err(|err| { CmdError::Failure(format!( - "failed to parse `{CONFIG_PG}/{PROP_ID}` ({prop_id:?}) as a UUID: {err}" + "failed to parse `{CONFIG_PG}/{PROP_ID}` \ + ({prop_id:?}) as a UUID: {err}" )) })?; - let prop_rack_id = config - .get_property(PROP_RACK_ID) - .map_err(|err| Error::GetProperty { prop: PROP_RACK_ID, err })? - .ok_or_else(|| Error::MissingProperty { prop: PROP_RACK_ID })? - .value() - .map_err(|err| Error::GetValue { prop: PROP_RACK_ID, err })? - .ok_or(Error::MissingValue { prop: PROP_RACK_ID })? - .as_string() - .map_err(|err| Error::ValueAsString { prop: PROP_RACK_ID, err })?; + let prop_rack_id = + config.value_as_string(PROP_RACK_ID).map_err(scf_to_cmd_err)?; - let rack_id = if prop_rack_id.as_str() == "unknown" { + let rack_id = if prop_rack_id == "unknown" { None } else { Some(Uuid::try_parse(&prop_rack_id).map_err(|err| { CmdError::Failure(format!( - "failed to parse `{CONFIG_PG}/{PROP_RACK_ID}` ({prop_rack_id:?}) as a UUID: {err}" + "failed to parse `{CONFIG_PG}/{PROP_RACK_ID}` \ + ({prop_rack_id:?}) as a UUID: {err}" )) })?) }; - let prop_addr = config - .get_property(PROP_ADDR) - .map_err(|err| Error::GetProperty { prop: PROP_ADDR, err })? - .ok_or_else(|| Error::MissingProperty { prop: PROP_ADDR })?; + let prop_addr = + config.values_as_strings(PROP_ADDR).map_err(scf_to_cmd_err)?; - let mut addresses = Vec::new(); + let mut addresses = Vec::with_capacity(prop_addr.len()); - for value in prop_addr - .values() - .map_err(|err| Error::GetValues { prop: PROP_ADDR, err })? - { - let addr = value - .map_err(|err| Error::GetValue { prop: PROP_ADDR, err })? - .as_string() - .map_err(|err| Error::ValueAsString { prop: PROP_ADDR, err })?; - - addresses.push(addr.parse().map_err(|err| CmdError::Failure(format!( - "failed to parse `{CONFIG_PG}/{PROP_ADDR}` ({addr:?}) as a socket address: {err}" - )))?); + for addr in prop_addr { + addresses.push(addr.parse().map_err(|err| { + CmdError::Failure(format!( + "failed to parse `{CONFIG_PG}/{PROP_ADDR}` \ + ({addr:?}) as a socket address: {err}" + )) + })?); } if addresses.is_empty() { diff --git a/gateway/src/http_entrypoints.rs b/gateway/src/http_entrypoints.rs index e51f7509a5..2db6121f1d 100644 --- a/gateway/src/http_entrypoints.rs +++ b/gateway/src/http_entrypoints.rs @@ -14,6 +14,7 @@ use self::conversions::component_from_str; use crate::error::SpCommsError; use crate::http_err_with_message; use crate::ServerContext; +use base64::Engine; use dropshot::endpoint; use dropshot::ApiDescription; use dropshot::HttpError; @@ -27,6 +28,7 @@ use dropshot::UntypedBody; use dropshot::WebsocketEndpointResult; use dropshot::WebsocketUpgrade; use futures::TryFutureExt; +use gateway_messages::SpComponent; use gateway_sp_comms::HostPhase2Provider; use omicron_common::update::ArtifactHash; use schemars::JsonSchema; @@ -118,6 +120,70 @@ pub struct RotImageDetails { pub version: ImageVersion, } +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +pub struct RotCmpa { + pub base64_data: String, +} + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +#[serde(tag = "slot", rename_all = "snake_case")] +pub enum RotCfpaSlot { + Active, + Inactive, + Scratch, +} + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +pub struct GetCfpaParams { + pub slot: RotCfpaSlot, +} + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +pub struct RotCfpa { + pub base64_data: String, + pub slot: RotCfpaSlot, +} + #[derive( Debug, Clone, @@ -963,6 +1029,75 @@ async fn sp_component_update_abort( Ok(HttpResponseUpdatedNoContent {}) } +/// Read the CMPA from a root of trust. +/// +/// This endpoint is only valid for the `rot` component. +#[endpoint { + method = GET, + path = "/sp/{type}/{slot}/component/{component}/cmpa", +}] +async fn sp_rot_cmpa_get( + rqctx: RequestContext>, + path: Path, +) -> Result, HttpError> { + let apictx = rqctx.context(); + + let PathSpComponent { sp, component } = path.into_inner(); + + // Ensure the caller knows they're asking for the RoT + if component_from_str(&component)? != SpComponent::ROT { + return Err(HttpError::for_bad_request( + Some("RequestUnsupportedForComponent".to_string()), + "Only the RoT has a CFPA".into(), + )); + } + + let sp = apictx.mgmt_switch.sp(sp.into())?; + let data = sp.read_rot_cmpa().await.map_err(SpCommsError::from)?; + + let base64_data = base64::engine::general_purpose::STANDARD.encode(data); + + Ok(HttpResponseOk(RotCmpa { base64_data })) +} + +/// Read the requested CFPA slot from a root of trust. +/// +/// This endpoint is only valid for the `rot` component. +#[endpoint { + method = GET, + path = "/sp/{type}/{slot}/component/{component}/cfpa", +}] +async fn sp_rot_cfpa_get( + rqctx: RequestContext>, + path: Path, + params: TypedBody, +) -> Result, HttpError> { + let apictx = rqctx.context(); + + let PathSpComponent { sp, component } = path.into_inner(); + let GetCfpaParams { slot } = params.into_inner(); + + // Ensure the caller knows they're asking for the RoT + if component_from_str(&component)? != SpComponent::ROT { + return Err(HttpError::for_bad_request( + Some("RequestUnsupportedForComponent".to_string()), + "Only the RoT has a CFPA".into(), + )); + } + + let sp = apictx.mgmt_switch.sp(sp.into())?; + let data = match slot { + RotCfpaSlot::Active => sp.read_rot_active_cfpa().await, + RotCfpaSlot::Inactive => sp.read_rot_inactive_cfpa().await, + RotCfpaSlot::Scratch => sp.read_rot_scratch_cfpa().await, + } + .map_err(SpCommsError::from)?; + + let base64_data = base64::engine::general_purpose::STANDARD.encode(data); + + Ok(HttpResponseOk(RotCfpa { base64_data, slot })) +} + /// List SPs via Ignition /// /// Retreive information for all SPs via the Ignition controller. This is lower @@ -1319,6 +1454,8 @@ pub fn api() -> GatewayApiDescription { api.register(sp_component_update)?; api.register(sp_component_update_status)?; api.register(sp_component_update_abort)?; + api.register(sp_rot_cmpa_get)?; + api.register(sp_rot_cfpa_get)?; api.register(sp_host_phase2_progress_get)?; api.register(sp_host_phase2_progress_delete)?; api.register(ignition_list)?; diff --git a/illumos-utils/Cargo.toml b/illumos-utils/Cargo.toml index e521b54d02..a291a15e78 100644 --- a/illumos-utils/Cargo.toml +++ b/illumos-utils/Cargo.toml @@ -12,6 +12,7 @@ bhyve_api.workspace = true byteorder.workspace = true camino.workspace = true cfg-if.workspace = true +crucible-smf.workspace = true futures.workspace = true ipnetwork.workspace = true libc.workspace = true diff --git a/illumos-utils/src/destructor.rs b/illumos-utils/src/destructor.rs index e019f2562f..ccc5b15486 100644 --- a/illumos-utils/src/destructor.rs +++ b/illumos-utils/src/destructor.rs @@ -21,7 +21,7 @@ use tokio::sync::mpsc; type SharedBoxFuture = Shared + Send>>>; -/// Future stored within [Destructor]. +/// Future stored within [`Destructor`]. struct ShutdownWaitFuture(SharedBoxFuture>); impl Future for ShutdownWaitFuture { diff --git a/illumos-utils/src/lib.rs b/illumos-utils/src/lib.rs index 1d585ee786..345f097ae2 100644 --- a/illumos-utils/src/lib.rs +++ b/illumos-utils/src/lib.rs @@ -17,6 +17,7 @@ pub mod libc; pub mod link; pub mod opte; pub mod running_zone; +pub mod scf; pub mod svc; pub mod vmm_reservoir; pub mod zfs; diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 10e2a45d83..710e783181 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -25,6 +25,7 @@ pub use port_manager::PortTicket; use ipnetwork::IpNetwork; use macaddr::MacAddr6; pub use oxide_vpc::api::BoundaryServices; +pub use oxide_vpc::api::DhcpCfg; pub use oxide_vpc::api::Vni; use std::net::IpAddr; diff --git a/illumos-utils/src/opte/params.rs b/illumos-utils/src/opte/params.rs index 4df437546c..df1f33cb92 100644 --- a/illumos-utils/src/opte/params.rs +++ b/illumos-utils/src/opte/params.rs @@ -50,3 +50,26 @@ pub struct DeleteVirtualNetworkInterfaceHost { /// be deleted. pub vni: external::Vni, } + +/// DHCP configuration for a port +/// +/// Not present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we +/// use `InstanceRuntimeState::hostname` for this value. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct DhcpConfig { + /// DNS servers to send to the instance + /// + /// (DHCPv4 option 6; DHCPv6 option 23) + pub dns_servers: Vec, + + /// DNS zone this instance's hostname belongs to (e.g. the `project.example` + /// part of `instance1.project.example`) + /// + /// (DHCPv4 option 15; used in DHCPv6 option 39) + pub host_domain: Option, + + /// DNS search domains + /// + /// (DHCPv4 option 119; DHCPv6 option 24) + pub search_domains: Vec, +} diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 893db9a6ed..f0a8d8d839 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -19,6 +19,7 @@ use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_common::api::internal::shared::SourceNatConfig; use oxide_vpc::api::AddRouterEntryReq; +use oxide_vpc::api::DhcpCfg; use oxide_vpc::api::IpCfg; use oxide_vpc::api::IpCidr; use oxide_vpc::api::Ipv4Cfg; @@ -100,6 +101,7 @@ impl PortManager { source_nat: Option, external_ips: &[IpAddr], firewall_rules: &[VpcFirewallRule], + dhcp_config: DhcpCfg, ) -> Result<(Port, PortTicket), Error> { let mac = *nic.mac; let vni = Vni::new(nic.vni).unwrap(); @@ -205,8 +207,6 @@ impl PortManager { vni, phys_ip: self.inner.underlay_ip.into(), boundary_services, - // TODO-completeness (#2153): Plumb domain search list - domain_list: vec![], }; // Create the xde device. @@ -227,11 +227,17 @@ impl PortManager { "Creating xde device"; "port_name" => &port_name, "vpc_cfg" => ?&vpc_cfg, + "dhcp_config" => ?&dhcp_config, ); #[cfg(target_os = "illumos")] let hdl = { let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?; - hdl.create_xde(&port_name, vpc_cfg, /* passthru = */ false)?; + hdl.create_xde( + &port_name, + vpc_cfg, + dhcp_config, + /* passthru = */ false, + )?; hdl }; diff --git a/illumos-utils/src/scf.rs b/illumos-utils/src/scf.rs new file mode 100644 index 0000000000..a691146531 --- /dev/null +++ b/illumos-utils/src/scf.rs @@ -0,0 +1,151 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Utilities for accessing SMF properties. + +pub use crucible_smf::ScfError as InnerScfError; + +#[derive(Debug, thiserror::Error)] +pub enum ScfError { + #[error("failed to create scf handle: {0}")] + ScfHandle(InnerScfError), + #[error("failed to get self smf instance: {0}")] + SelfInstance(InnerScfError), + #[error("failed to get self running snapshot: {0}")] + RunningSnapshot(InnerScfError), + #[error("failed to get propertygroup `{group}`: {err}")] + GetPg { group: &'static str, err: InnerScfError }, + #[error("missing propertygroup `{group}`")] + MissingPg { group: &'static str }, + #[error("failed to get property `{group}/{prop}`: {err}")] + GetProperty { group: &'static str, prop: &'static str, err: InnerScfError }, + #[error("missing property `{group}/{prop}`")] + MissingProperty { group: &'static str, prop: &'static str }, + #[error("failed to get value for `{group}/{prop}`: {err}")] + GetValue { group: &'static str, prop: &'static str, err: InnerScfError }, + #[error("failed to get values for `{group}/{prop}`: {err}")] + GetValues { group: &'static str, prop: &'static str, err: InnerScfError }, + #[error("failed to get value for `{group}/{prop}`")] + MissingValue { group: &'static str, prop: &'static str }, + #[error("failed to get `{group}/{prop} as a string: {err}")] + ValueAsString { + group: &'static str, + prop: &'static str, + err: InnerScfError, + }, +} + +pub struct ScfHandle { + inner: crucible_smf::Scf, +} + +impl ScfHandle { + pub fn new() -> Result { + match crucible_smf::Scf::new() { + Ok(inner) => Ok(Self { inner }), + Err(err) => Err(ScfError::ScfHandle(err)), + } + } + + pub fn self_instance(&self) -> Result, ScfError> { + match self.inner.get_self_instance() { + Ok(inner) => Ok(ScfInstance { inner }), + Err(err) => Err(ScfError::SelfInstance(err)), + } + } +} + +pub struct ScfInstance<'a> { + inner: crucible_smf::Instance<'a>, +} + +impl ScfInstance<'_> { + pub fn running_snapshot(&self) -> Result, ScfError> { + match self.inner.get_running_snapshot() { + Ok(inner) => Ok(ScfSnapshot { inner }), + Err(err) => Err(ScfError::RunningSnapshot(err)), + } + } +} + +pub struct ScfSnapshot<'a> { + inner: crucible_smf::Snapshot<'a>, +} + +impl ScfSnapshot<'_> { + pub fn property_group( + &self, + group: &'static str, + ) -> Result, ScfError> { + match self.inner.get_pg(group) { + Ok(Some(inner)) => Ok(ScfPropertyGroup { group, inner }), + Ok(None) => Err(ScfError::MissingPg { group }), + Err(err) => Err(ScfError::GetPg { group, err }), + } + } +} + +pub struct ScfPropertyGroup<'a> { + group: &'static str, + inner: crucible_smf::PropertyGroup<'a>, +} + +impl ScfPropertyGroup<'_> { + fn property<'a>( + &'a self, + prop: &'static str, + ) -> Result, ScfError> { + match self.inner.get_property(prop) { + Ok(Some(prop)) => Ok(prop), + Ok(None) => { + Err(ScfError::MissingProperty { group: self.group, prop }) + } + Err(err) => { + Err(ScfError::GetProperty { group: self.group, prop, err }) + } + } + } + + pub fn value_as_string( + &self, + prop: &'static str, + ) -> Result { + let inner = self.property(prop)?; + let value = inner + .value() + .map_err(|err| ScfError::GetValue { group: self.group, prop, err })? + .ok_or(ScfError::MissingValue { group: self.group, prop })?; + value.as_string().map_err(|err| ScfError::ValueAsString { + group: self.group, + prop, + err, + }) + } + + pub fn values_as_strings( + &self, + prop: &'static str, + ) -> Result, ScfError> { + let inner = self.property(prop)?; + let values = inner.values().map_err(|err| ScfError::GetValues { + group: self.group, + prop, + err, + })?; + values + .map(|value| { + let value = value.map_err(|err| ScfError::GetValue { + group: self.group, + prop, + err, + })?; + value.as_string().map_err(|err| ScfError::ValueAsString { + group: self.group, + prop, + err, + }) + }) + .collect() + } +} diff --git a/installinator/src/bootstrap.rs b/installinator/src/bootstrap.rs index 2854293d8a..71c76809db 100644 --- a/installinator/src/bootstrap.rs +++ b/installinator/src/bootstrap.rs @@ -20,7 +20,7 @@ use sled_hardware::underlay::BootstrapInterface; use slog::info; use slog::Logger; -const MG_DDM_SERVICE_FMRI: &str = "svc:/system/illumos/mg-ddm"; +const MG_DDM_SERVICE_FMRI: &str = "svc:/oxide/mg-ddm"; const MG_DDM_MANIFEST_PATH: &str = "/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"; // TODO-cleanup The implementation of this function is heavily derived from diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index 9c06aeac77..9bec14664c 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -104,7 +104,7 @@ impl DebugDiscoverOpts { /// Options shared by both [`DebugDiscoverOpts`] and [`InstallOpts`]. #[derive(Debug, Args)] struct DiscoverOpts { - /// The mechanism by which to discover peers: bootstrap or list:[::1]:8000 + /// The mechanism by which to discover peers: bootstrap or `list:[::1]:8000` #[clap(long, default_value_t = DiscoveryMechanism::Bootstrap)] mechanism: DiscoveryMechanism, } diff --git a/internal-dns/src/config.rs b/internal-dns/src/config.rs index e5272cd23a..86dd6e802e 100644 --- a/internal-dns/src/config.rs +++ b/internal-dns/src/config.rs @@ -63,8 +63,9 @@ use crate::names::{ServiceName, DNS_ZONE}; use anyhow::{anyhow, ensure}; use dns_service_client::types::{DnsConfigParams, DnsConfigZone, DnsRecord}; +use omicron_common::api::internal::shared::SwitchLocation; use std::collections::BTreeMap; -use std::net::Ipv6Addr; +use std::net::{Ipv6Addr, SocketAddrV6}; use uuid::Uuid; /// Zones that can be referenced within the internal DNS system. @@ -136,6 +137,8 @@ pub struct DnsConfigBuilder { /// network sleds: BTreeMap, + scrimlets: BTreeMap, + /// set of hosts of type "zone" that have been configured so far, mapping /// each zone's unique uuid to its sole IPv6 address on the control plane /// network @@ -175,6 +178,7 @@ impl DnsConfigBuilder { DnsConfigBuilder { sleds: BTreeMap::new(), zones: BTreeMap::new(), + scrimlets: BTreeMap::new(), service_instances_zones: BTreeMap::new(), service_instances_sleds: BTreeMap::new(), } @@ -205,6 +209,15 @@ impl DnsConfigBuilder { } } + pub fn host_scrimlet( + &mut self, + switch_location: SwitchLocation, + addr: SocketAddrV6, + ) -> anyhow::Result<()> { + self.scrimlets.insert(switch_location, addr); + Ok(()) + } + /// Add a new dendrite host of type "zone" to the configuration /// /// Returns a [`Zone`] that can be used with [`Self::service_backend_zone()`] to @@ -351,6 +364,23 @@ impl DnsConfigBuilder { (zone.dns_name(), vec![DnsRecord::Aaaa(zone_ip)]) }); + let scrimlet_srv_records = + self.scrimlets.clone().into_iter().map(|(location, addr)| { + let srv = DnsRecord::Srv(dns_service_client::types::Srv { + prio: 0, + weight: 0, + port: addr.port(), + target: format!("{location}.scrimlet.{}", DNS_ZONE), + }); + (ServiceName::Scrimlet(location).dns_name(), vec![srv]) + }); + + let scrimlet_aaaa_records = + self.scrimlets.into_iter().map(|(location, addr)| { + let aaaa = DnsRecord::Aaaa(*addr.ip()); + (format!("{location}.scrimlet"), vec![aaaa]) + }); + // Assemble the set of SRV records, which implicitly point back at // zones' AAAA records. let srv_records_zones = self.service_instances_zones.into_iter().map( @@ -399,6 +429,8 @@ impl DnsConfigBuilder { .chain(zone_records) .chain(srv_records_sleds) .chain(srv_records_zones) + .chain(scrimlet_aaaa_records) + .chain(scrimlet_srv_records) .collect(); DnsConfigParams { diff --git a/internal-dns/src/names.rs b/internal-dns/src/names.rs index 44ed9228e2..e0c9b79555 100644 --- a/internal-dns/src/names.rs +++ b/internal-dns/src/names.rs @@ -4,6 +4,7 @@ //! Well-known DNS names and related types for internal DNS (see RFD 248) +use omicron_common::api::internal::shared::SwitchLocation; use uuid::Uuid; /// Name for the control plane DNS zone @@ -32,7 +33,9 @@ pub enum ServiceName { Crucible(Uuid), BoundaryNtp, InternalNtp, - Maghemite, + Maghemite, //TODO change to Dpd - maghemite has several services. + Mgd, + Scrimlet(SwitchLocation), } impl ServiceName { @@ -55,6 +58,8 @@ impl ServiceName { ServiceName::BoundaryNtp => "boundary-ntp", ServiceName::InternalNtp => "internal-ntp", ServiceName::Maghemite => "maghemite", + ServiceName::Mgd => "mgd", + ServiceName::Scrimlet(_) => "scrimlet", } } @@ -76,7 +81,8 @@ impl ServiceName { | ServiceName::CruciblePantry | ServiceName::BoundaryNtp | ServiceName::InternalNtp - | ServiceName::Maghemite => { + | ServiceName::Maghemite + | ServiceName::Mgd => { format!("_{}._tcp", self.service_kind()) } ServiceName::SledAgent(id) => { @@ -85,6 +91,9 @@ impl ServiceName { ServiceName::Crucible(id) => { format!("_{}._tcp.{}", self.service_kind(), id) } + ServiceName::Scrimlet(location) => { + format!("_{location}._scrimlet._tcp") + } } } diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 65a16b0d35..feb25eb1f1 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -22,6 +22,7 @@ crucible-agent-client.workspace = true crucible-pantry-client.workspace = true dns-service-client.workspace = true dpd-client.workspace = true +mg-admin-client.workspace = true dropshot.workspace = true fatfs.workspace = true futures.workspace = true diff --git a/nexus/db-macros/src/lookup.rs b/nexus/db-macros/src/lookup.rs index 38cab15e30..f2362f5bc5 100644 --- a/nexus/db-macros/src/lookup.rs +++ b/nexus/db-macros/src/lookup.rs @@ -15,7 +15,7 @@ use std::ops::Deref; // INPUT (arguments to the macro) // -/// Arguments for [`lookup_resource!`] +/// Arguments for [`super::lookup_resource!`] // NOTE: this is only "pub" for the `cargo doc` link on [`lookup_resource!`]. #[derive(serde::Deserialize)] pub struct Input { @@ -167,7 +167,7 @@ impl Resource { // MACRO IMPLEMENTATION // -/// Implementation of [`lookup_resource!`] +/// Implementation of [`super::lookup_resource!`] pub fn lookup_resource( raw_input: TokenStream, ) -> Result { diff --git a/nexus/db-model/src/bgp.rs b/nexus/db-model/src/bgp.rs index 532b9cce36..cc9ebfb4f5 100644 --- a/nexus/db-model/src/bgp.rs +++ b/nexus/db-model/src/bgp.rs @@ -6,8 +6,10 @@ use crate::schema::{bgp_announce_set, bgp_announcement, bgp_config}; use crate::SqlU32; use db_macros::Resource; use ipnetwork::IpNetwork; +use nexus_types::external_api::params; use nexus_types::identity::Resource; use omicron_common::api::external; +use omicron_common::api::external::IdentityMetadataCreateParams; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -26,6 +28,7 @@ pub struct BgpConfig { #[diesel(embed)] pub identity: BgpConfigIdentity, pub asn: SqlU32, + pub bgp_announce_set_id: Uuid, pub vrf: Option, } @@ -39,6 +42,26 @@ impl Into for BgpConfig { } } +impl BgpConfig { + pub fn from_config_create( + c: ¶ms::BgpConfigCreate, + bgp_announce_set_id: Uuid, + ) -> BgpConfig { + BgpConfig { + identity: BgpConfigIdentity::new( + Uuid::new_v4(), + IdentityMetadataCreateParams { + name: c.identity.name.clone(), + description: c.identity.description.clone(), + }, + ), + asn: c.asn.into(), + bgp_announce_set_id, + vrf: c.vrf.as_ref().map(|x| x.to_string()), + } + } +} + #[derive( Queryable, Insertable, @@ -55,6 +78,20 @@ pub struct BgpAnnounceSet { pub identity: BgpAnnounceSetIdentity, } +impl From for BgpAnnounceSet { + fn from(x: params::BgpAnnounceSetCreate) -> BgpAnnounceSet { + BgpAnnounceSet { + identity: BgpAnnounceSetIdentity::new( + Uuid::new_v4(), + IdentityMetadataCreateParams { + name: x.identity.name.clone(), + description: x.identity.description.clone(), + }, + ), + } + } +} + impl Into for BgpAnnounceSet { fn into(self) -> external::BgpAnnounceSet { external::BgpAnnounceSet { identity: self.identity() } diff --git a/nexus/db-model/src/bootstore.rs b/nexus/db-model/src/bootstore.rs new file mode 100644 index 0000000000..38afd37f54 --- /dev/null +++ b/nexus/db-model/src/bootstore.rs @@ -0,0 +1,13 @@ +use crate::schema::bootstore_keys; +use serde::{Deserialize, Serialize}; + +pub const NETWORK_KEY: &str = "network_key"; + +#[derive( + Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, +)] +#[diesel(table_name = bootstore_keys)] +pub struct BootstoreKeys { + pub key: String, + pub generation: i64, +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index a424551e7b..7aa8a6b076 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -12,6 +12,7 @@ extern crate newtype_derive; mod address_lot; mod bgp; mod block_size; +mod bootstore; mod bytecount; mod certificate; mod collection; @@ -101,6 +102,7 @@ pub use self::unsigned::*; pub use address_lot::*; pub use bgp::*; pub use block_size::*; +pub use bootstore::*; pub use bytecount::*; pub use certificate::*; pub use collection::*; diff --git a/nexus/db-model/src/rack.rs b/nexus/db-model/src/rack.rs index 0f1ef2a853..580ec155b4 100644 --- a/nexus/db-model/src/rack.rs +++ b/nexus/db-model/src/rack.rs @@ -4,6 +4,7 @@ use crate::schema::rack; use db_macros::Asset; +use ipnetwork::IpNetwork; use nexus_types::{external_api::views, identity::Asset}; use uuid::Uuid; @@ -15,6 +16,7 @@ pub struct Rack { pub identity: RackIdentity, pub initialized: bool, pub tuf_base_url: Option, + pub rack_subnet: Option, } impl Rack { @@ -23,6 +25,7 @@ impl Rack { identity: RackIdentity::new(id), initialized: false, tuf_base_url: None, + rack_subnet: None, } } } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index b2c957205d..cefdd9f006 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -144,6 +144,8 @@ table! { lldp_service_config_id -> Uuid, link_name -> Text, mtu -> Int4, + fec -> crate::SwitchLinkFecEnum, + speed -> crate::SwitchLinkSpeedEnum, } } @@ -188,7 +190,7 @@ table! { } table! { - switch_port_settings_route_config (port_settings_id, interface_name, dst, gw, vid) { + switch_port_settings_route_config (port_settings_id, interface_name, dst, gw) { port_settings_id -> Uuid, interface_name -> Text, dst -> Inet, @@ -200,10 +202,14 @@ table! { table! { switch_port_settings_bgp_peer_config (port_settings_id, interface_name, addr) { port_settings_id -> Uuid, - bgp_announce_set_id -> Uuid, bgp_config_id -> Uuid, interface_name -> Text, addr -> Inet, + hold_time -> Int8, + idle_hold_time -> Int8, + delay_open -> Int8, + connect_retry -> Int8, + keepalive -> Int8, } } @@ -216,6 +222,7 @@ table! { time_modified -> Timestamptz, time_deleted -> Nullable, asn -> Int8, + bgp_announce_set_id -> Uuid, vrf -> Nullable, } } @@ -673,6 +680,7 @@ table! { time_modified -> Timestamptz, initialized -> Bool, tuf_base_url -> Nullable, + rack_subnet -> Nullable, } } @@ -812,6 +820,11 @@ table! { } } +allow_tables_to_appear_in_same_query! { + virtual_provisioning_resource, + instance +} + table! { zpool (id) { id -> Uuid, @@ -1208,6 +1221,13 @@ table! { } } +table! { + bootstore_keys (key, generation) { + key -> Text, + generation -> Int8, + } +} + table! { db_metadata (singleton) { singleton -> Bool, @@ -1223,7 +1243,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(7, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(8, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/nexus/db-model/src/service_kind.rs b/nexus/db-model/src/service_kind.rs index c2598434d5..4210c3ee20 100644 --- a/nexus/db-model/src/service_kind.rs +++ b/nexus/db-model/src/service_kind.rs @@ -30,6 +30,7 @@ impl_enum_type!( Oximeter => b"oximeter" Tfport => b"tfport" Ntp => b"ntp" + Mgd => b"mgd" ); impl TryFrom for ServiceUsingCertificate { @@ -88,6 +89,7 @@ impl From for ServiceKind { | internal_api::params::ServiceKind::InternalNtp => { ServiceKind::Ntp } + internal_api::params::ServiceKind::Mgd => ServiceKind::Mgd, } } } diff --git a/nexus/db-model/src/switch_interface.rs b/nexus/db-model/src/switch_interface.rs index 9ac7e4323a..f0c4b91de6 100644 --- a/nexus/db-model/src/switch_interface.rs +++ b/nexus/db-model/src/switch_interface.rs @@ -64,7 +64,14 @@ impl Into for DbSwitchInterfaceKind { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_vlan_interface_config)] pub struct SwitchVlanInterfaceConfig { diff --git a/nexus/db-model/src/switch_port.rs b/nexus/db-model/src/switch_port.rs index e9c0697450..44588899b6 100644 --- a/nexus/db-model/src/switch_port.rs +++ b/nexus/db-model/src/switch_port.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::impl_enum_type; use crate::schema::{ lldp_config, lldp_service_config, switch_port, switch_port_settings, switch_port_settings_address_config, switch_port_settings_bgp_peer_config, @@ -11,11 +10,14 @@ use crate::schema::{ switch_port_settings_port_config, switch_port_settings_route_config, }; use crate::SqlU16; +use crate::{impl_enum_type, SqlU32}; use db_macros::Resource; +use diesel::AsChangeset; use ipnetwork::IpNetwork; use nexus_types::external_api::params; use nexus_types::identity::Resource; use omicron_common::api::external; +use omicron_common::api::internal::shared::{PortFec, PortSpeed}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -42,6 +44,110 @@ impl_enum_type!( Sfp28x4 => b"Sfp28x4" ); +impl_enum_type!( + #[derive(SqlType, Debug, Clone, Copy)] + #[diesel(postgres_type(name = "switch_link_fec"))] + pub struct SwitchLinkFecEnum; + + #[derive( + Clone, + Copy, + Debug, + AsExpression, + FromSqlRow, + PartialEq, + Serialize, + Deserialize + )] + #[diesel(sql_type = SwitchLinkFecEnum)] + pub enum SwitchLinkFec; + + Firecode => b"Firecode" + None => b"None" + Rs => b"Rs" +); + +impl_enum_type!( + #[derive(SqlType, Debug, Clone, Copy)] + #[diesel(postgres_type(name = "switch_link_speed"))] + pub struct SwitchLinkSpeedEnum; + + #[derive( + Clone, + Copy, + Debug, + AsExpression, + FromSqlRow, + PartialEq, + Serialize, + Deserialize + )] + #[diesel(sql_type = SwitchLinkSpeedEnum)] + pub enum SwitchLinkSpeed; + + Speed0G => b"0G" + Speed1G => b"1G" + Speed10G => b"10G" + Speed25G => b"25G" + Speed40G => b"40G" + Speed50G => b"50G" + Speed100G => b"100G" + Speed200G => b"200G" + Speed400G => b"400G" +); + +impl From for PortFec { + fn from(value: SwitchLinkFec) -> Self { + match value { + SwitchLinkFec::Firecode => PortFec::Firecode, + SwitchLinkFec::None => PortFec::None, + SwitchLinkFec::Rs => PortFec::Rs, + } + } +} + +impl From for SwitchLinkFec { + fn from(value: params::LinkFec) -> Self { + match value { + params::LinkFec::Firecode => SwitchLinkFec::Firecode, + params::LinkFec::None => SwitchLinkFec::None, + params::LinkFec::Rs => SwitchLinkFec::Rs, + } + } +} + +impl From for PortSpeed { + fn from(value: SwitchLinkSpeed) -> Self { + match value { + SwitchLinkSpeed::Speed0G => PortSpeed::Speed0G, + SwitchLinkSpeed::Speed1G => PortSpeed::Speed1G, + SwitchLinkSpeed::Speed10G => PortSpeed::Speed10G, + SwitchLinkSpeed::Speed25G => PortSpeed::Speed25G, + SwitchLinkSpeed::Speed40G => PortSpeed::Speed40G, + SwitchLinkSpeed::Speed50G => PortSpeed::Speed50G, + SwitchLinkSpeed::Speed100G => PortSpeed::Speed100G, + SwitchLinkSpeed::Speed200G => PortSpeed::Speed200G, + SwitchLinkSpeed::Speed400G => PortSpeed::Speed400G, + } + } +} + +impl From for SwitchLinkSpeed { + fn from(value: params::LinkSpeed) -> Self { + match value { + params::LinkSpeed::Speed0G => SwitchLinkSpeed::Speed0G, + params::LinkSpeed::Speed1G => SwitchLinkSpeed::Speed1G, + params::LinkSpeed::Speed10G => SwitchLinkSpeed::Speed10G, + params::LinkSpeed::Speed25G => SwitchLinkSpeed::Speed25G, + params::LinkSpeed::Speed40G => SwitchLinkSpeed::Speed40G, + params::LinkSpeed::Speed50G => SwitchLinkSpeed::Speed50G, + params::LinkSpeed::Speed100G => SwitchLinkSpeed::Speed100G, + params::LinkSpeed::Speed200G => SwitchLinkSpeed::Speed200G, + params::LinkSpeed::Speed400G => SwitchLinkSpeed::Speed400G, + } + } +} + impl From for SwitchPortGeometry { fn from(g: params::SwitchPortGeometry) -> Self { match g { @@ -140,14 +246,21 @@ pub struct SwitchPortSettings { } impl SwitchPortSettings { - pub fn new(id: &external::IdentityMetadataCreateParams) -> Self { + pub fn new(meta: &external::IdentityMetadataCreateParams) -> Self { Self { identity: SwitchPortSettingsIdentity::new( Uuid::new_v4(), - id.clone(), + meta.clone(), ), } } + + pub fn with_id( + id: Uuid, + meta: &external::IdentityMetadataCreateParams, + ) -> Self { + Self { identity: SwitchPortSettingsIdentity::new(id, meta.clone()) } + } } impl Into for SwitchPortSettings { @@ -225,7 +338,14 @@ impl Into for SwitchPortConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_link_config)] pub struct SwitchPortLinkConfig { @@ -233,6 +353,8 @@ pub struct SwitchPortLinkConfig { pub lldp_service_config_id: Uuid, pub link_name: String, pub mtu: SqlU16, + pub fec: SwitchLinkFec, + pub speed: SwitchLinkSpeed, } impl SwitchPortLinkConfig { @@ -241,11 +363,15 @@ impl SwitchPortLinkConfig { lldp_service_config_id: Uuid, link_name: String, mtu: u16, + fec: SwitchLinkFec, + speed: SwitchLinkSpeed, ) -> Self { Self { port_settings_id, lldp_service_config_id, link_name, + fec, + speed, mtu: mtu.into(), } } @@ -263,7 +389,14 @@ impl Into for SwitchPortLinkConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = lldp_service_config)] pub struct LldpServiceConfig { @@ -321,7 +454,14 @@ impl Into for LldpConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_interface_config)] pub struct SwitchInterfaceConfig { @@ -362,7 +502,14 @@ impl Into for SwitchInterfaceConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_route_config)] pub struct SwitchPortRouteConfig { @@ -398,31 +545,51 @@ impl Into for SwitchPortRouteConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_bgp_peer_config)] pub struct SwitchPortBgpPeerConfig { pub port_settings_id: Uuid, - pub bgp_announce_set_id: Uuid, pub bgp_config_id: Uuid, pub interface_name: String, pub addr: IpNetwork, + pub hold_time: SqlU32, + pub idle_hold_time: SqlU32, + pub delay_open: SqlU32, + pub connect_retry: SqlU32, + pub keepalive: SqlU32, } impl SwitchPortBgpPeerConfig { + #[allow(clippy::too_many_arguments)] pub fn new( port_settings_id: Uuid, - bgp_announce_set_id: Uuid, bgp_config_id: Uuid, interface_name: String, addr: IpNetwork, + hold_time: SqlU32, + idle_hold_time: SqlU32, + delay_open: SqlU32, + connect_retry: SqlU32, + keepalive: SqlU32, ) -> Self { Self { port_settings_id, - bgp_announce_set_id, bgp_config_id, interface_name, addr, + hold_time, + idle_hold_time, + delay_open, + connect_retry, + keepalive, } } } @@ -431,7 +598,6 @@ impl Into for SwitchPortBgpPeerConfig { fn into(self) -> external::SwitchPortBgpPeerConfig { external::SwitchPortBgpPeerConfig { port_settings_id: self.port_settings_id, - bgp_announce_set_id: self.bgp_announce_set_id, bgp_config_id: self.bgp_config_id, interface_name: self.interface_name.clone(), addr: self.addr.ip(), @@ -440,7 +606,14 @@ impl Into for SwitchPortBgpPeerConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_address_config)] pub struct SwitchPortAddressConfig { diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index 62adbe5bd2..5edf4f1e89 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -47,6 +47,7 @@ serde_urlencoded.workspace = true serde_with.workspace = true sled-agent-client.workspace = true slog.workspace = true +static_assertions.workspace = true steno.workspace = true thiserror.workspace = true tokio = { workspace = true, features = [ "full" ] } diff --git a/nexus/db-queries/src/db/datastore/bgp.rs b/nexus/db-queries/src/db/datastore/bgp.rs new file mode 100644 index 0000000000..ff314a2564 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/bgp.rs @@ -0,0 +1,351 @@ +use super::DataStore; +use crate::context::OpContext; +use crate::db; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::error::TransactionError; +use crate::db::model::Name; +use crate::db::model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig}; +use crate::db::pagination::paginated; +use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; +use chrono::Utc; +use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; +use nexus_types::external_api::params; +use nexus_types::identity::Resource; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::{ + CreateResult, DeleteResult, Error, ListResultVec, LookupResult, NameOrId, + ResourceType, +}; +use ref_cast::RefCast; +use uuid::Uuid; + +impl DataStore { + pub async fn bgp_config_set( + &self, + opctx: &OpContext, + config: ¶ms::BgpConfigCreate, + ) -> CreateResult { + use db::schema::bgp_config::dsl; + use db::schema::{ + bgp_announce_set, bgp_announce_set::dsl as announce_set_dsl, + }; + let pool = self.pool_connection_authorized(opctx).await?; + + pool.transaction_async(|conn| async move { + let id: Uuid = match &config.bgp_announce_set_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::time_deleted.is_null()) + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => *id, + }; + + let config = BgpConfig::from_config_create(config, id); + + let result = diesel::insert_into(dsl::bgp_config) + .values(config.clone()) + .returning(BgpConfig::as_returning()) + .get_result_async(&conn) + .await?; + Ok(result) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn bgp_config_delete( + &self, + opctx: &OpContext, + sel: ¶ms::BgpConfigSelector, + ) -> DeleteResult { + use db::schema::bgp_config; + use db::schema::bgp_config::dsl as bgp_config_dsl; + + use db::schema::switch_port_settings_bgp_peer_config as sps_bgp_peer_config; + use db::schema::switch_port_settings_bgp_peer_config::dsl as sps_bgp_peer_config_dsl; + + #[derive(Debug)] + enum BgpConfigDeleteError { + ConfigInUse, + } + type TxnError = TransactionError; + + let pool = self.pool_connection_authorized(opctx).await?; + pool.transaction_async(|conn| async move { + let name_or_id = sel.name_or_id.clone(); + + let id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => { + bgp_config_dsl::bgp_config + .filter(bgp_config::name.eq(name.to_string())) + .select(bgp_config::id) + .limit(1) + .first_async::(&conn) + .await? + } + }; + + let count = + sps_bgp_peer_config_dsl::switch_port_settings_bgp_peer_config + .filter(sps_bgp_peer_config::bgp_config_id.eq(id)) + .count() + .execute_async(&conn) + .await?; + + if count > 0 { + return Err(TxnError::CustomError( + BgpConfigDeleteError::ConfigInUse, + )); + } + + diesel::update(bgp_config_dsl::bgp_config) + .filter(bgp_config_dsl::id.eq(id)) + .set(bgp_config_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; + + Ok(()) + }) + .await + .map_err(|e| match e { + TxnError::CustomError(BgpConfigDeleteError::ConfigInUse) => { + Error::invalid_request("BGP config in use") + } + TxnError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + + pub async fn bgp_config_get( + &self, + opctx: &OpContext, + name_or_id: &NameOrId, + ) -> LookupResult { + use db::schema::bgp_config; + use db::schema::bgp_config::dsl; + let pool = self.pool_connection_authorized(opctx).await?; + + let name_or_id = name_or_id.clone(); + + let config = match name_or_id { + NameOrId::Name(name) => dsl::bgp_config + .filter(bgp_config::name.eq(name.to_string())) + .select(BgpConfig::as_select()) + .limit(1) + .first_async::(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), + NameOrId::Id(id) => dsl::bgp_config + .filter(bgp_config::id.eq(id)) + .select(BgpConfig::as_select()) + .limit(1) + .first_async::(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), + }?; + + Ok(config) + } + + pub async fn bgp_config_list( + &self, + opctx: &OpContext, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::bgp_config::dsl; + + let pool = self.pool_connection_authorized(opctx).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::bgp_config, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::bgp_config, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::time_deleted.is_null()) + .select(BgpConfig::as_select()) + .load_async(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn bgp_announce_list( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> ListResultVec { + use db::schema::{ + bgp_announce_set, bgp_announce_set::dsl as announce_set_dsl, + bgp_announcement::dsl as announce_dsl, + }; + + #[derive(Debug)] + enum BgpAnnounceListError { + AnnounceSetNotFound(Name), + } + type TxnError = TransactionError; + + let pool = self.pool_connection_authorized(opctx).await?; + pool.transaction_async(|conn| async move { + let name_or_id = sel.name_or_id.clone(); + + let announce_id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::time_deleted.is_null()) + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|_| { + TxnError::CustomError( + BgpAnnounceListError::AnnounceSetNotFound( + Name::from(name.clone()), + ), + ) + })?, + }; + + let result = announce_dsl::bgp_announcement + .filter(announce_dsl::announce_set_id.eq(announce_id)) + .select(BgpAnnouncement::as_select()) + .load_async(&conn) + .await?; + + Ok(result) + }) + .await + .map_err(|e| match e { + TxnError::CustomError( + BgpAnnounceListError::AnnounceSetNotFound(name), + ) => Error::not_found_by_name(ResourceType::BgpAnnounceSet, &name), + TxnError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + + pub async fn bgp_create_announce_set( + &self, + opctx: &OpContext, + announce: ¶ms::BgpAnnounceSetCreate, + ) -> CreateResult<(BgpAnnounceSet, Vec)> { + use db::schema::bgp_announce_set::dsl as announce_set_dsl; + use db::schema::bgp_announcement::dsl as bgp_announcement_dsl; + + let pool = self.pool_connection_authorized(opctx).await?; + pool.transaction_async(|conn| async move { + let bas: BgpAnnounceSet = announce.clone().into(); + + let db_as: BgpAnnounceSet = + diesel::insert_into(announce_set_dsl::bgp_announce_set) + .values(bas.clone()) + .returning(BgpAnnounceSet::as_returning()) + .get_result_async::(&conn) + .await?; + + let mut db_annoucements = Vec::new(); + for a in &announce.announcement { + let an = BgpAnnouncement { + announce_set_id: db_as.id(), + address_lot_block_id: bas.identity.id, + network: a.network.into(), + }; + let an = + diesel::insert_into(bgp_announcement_dsl::bgp_announcement) + .values(an.clone()) + .returning(BgpAnnouncement::as_returning()) + .get_result_async::(&conn) + .await?; + db_annoucements.push(an); + } + + Ok((db_as, db_annoucements)) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn bgp_delete_announce_set( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> DeleteResult { + use db::schema::bgp_announce_set; + use db::schema::bgp_announce_set::dsl as announce_set_dsl; + use db::schema::bgp_announcement::dsl as bgp_announcement_dsl; + + use db::schema::bgp_config; + use db::schema::bgp_config::dsl as bgp_config_dsl; + + #[derive(Debug)] + enum BgpAnnounceSetDeleteError { + AnnounceSetInUse, + } + type TxnError = TransactionError; + + let pool = self.pool_connection_authorized(opctx).await?; + let name_or_id = sel.name_or_id.clone(); + + pool.transaction_async(|conn| async move { + let id: Uuid = match name_or_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => id, + }; + + let count = bgp_config_dsl::bgp_config + .filter(bgp_config::bgp_announce_set_id.eq(id)) + .count() + .execute_async(&conn) + .await?; + + if count > 0 { + return Err(TxnError::CustomError( + BgpAnnounceSetDeleteError::AnnounceSetInUse, + )); + } + + diesel::update(announce_set_dsl::bgp_announce_set) + .filter(announce_set_dsl::id.eq(id)) + .set(announce_set_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; + + diesel::delete(bgp_announcement_dsl::bgp_announcement) + .filter(bgp_announcement_dsl::announce_set_id.eq(id)) + .execute_async(&conn) + .await?; + + Ok(()) + }) + .await + .map_err(|e| match e { + TxnError::CustomError( + BgpAnnounceSetDeleteError::AnnounceSetInUse, + ) => Error::invalid_request("BGP announce set in use"), + TxnError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } +} diff --git a/nexus/db-queries/src/db/datastore/bootstore.rs b/nexus/db-queries/src/db/datastore/bootstore.rs new file mode 100644 index 0000000000..44f7a2036e --- /dev/null +++ b/nexus/db-queries/src/db/datastore/bootstore.rs @@ -0,0 +1,37 @@ +use super::DataStore; +use crate::context::OpContext; +use crate::db; +use crate::db::error::{public_error_from_diesel, ErrorHandler}; +use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::ExpressionMethods; +use diesel::SelectableHelper; +use nexus_db_model::BootstoreKeys; +use omicron_common::api::external::LookupResult; + +impl DataStore { + pub async fn bump_bootstore_generation( + &self, + opctx: &OpContext, + key: String, + ) -> LookupResult { + use db::schema::bootstore_keys; + use db::schema::bootstore_keys::dsl; + + let conn = self.pool_connection_authorized(opctx).await?; + + let bks = diesel::insert_into(dsl::bootstore_keys) + .values(BootstoreKeys { + key: key.clone(), + generation: 2, // RSS starts with a generation of 1 + }) + .on_conflict(bootstore_keys::key) + .do_update() + .set(bootstore_keys::generation.eq(dsl::generation + 1)) + .returning(BootstoreKeys::as_returning()) + .get_result_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(bks.generation) + } +} diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs index 9e4e8b1a48..0ae61a7c38 100644 --- a/nexus/db-queries/src/db/datastore/db_metadata.rs +++ b/nexus/db-queries/src/db/datastore/db_metadata.rs @@ -25,6 +25,17 @@ use std::str::FromStr; pub const EARLIEST_SUPPORTED_VERSION: &'static str = "1.0.0"; +/// Describes a single file containing a schema change, as SQL. +pub struct SchemaUpgradeStep { + pub path: Utf8PathBuf, + pub sql: String, +} + +/// Describes a sequence of files containing schema changes. +pub struct SchemaUpgrade { + pub steps: Vec, +} + /// Reads a "version directory" and reads all SQL changes into /// a result Vec. /// @@ -34,7 +45,7 @@ pub const EARLIEST_SUPPORTED_VERSION: &'static str = "1.0.0"; /// These are sorted lexicographically. pub async fn all_sql_for_version_migration>( path: P, -) -> Result, String> { +) -> Result { let target_dir = path.as_ref(); let mut up_sqls = vec![]; let entries = target_dir @@ -54,13 +65,12 @@ pub async fn all_sql_for_version_migration>( } up_sqls.sort(); - let mut result = vec![]; + let mut result = SchemaUpgrade { steps: vec![] }; for path in up_sqls.into_iter() { - result.push( - tokio::fs::read_to_string(&path) - .await - .map_err(|e| format!("Cannot read {path}: {e}"))?, - ); + let sql = tokio::fs::read_to_string(&path) + .await + .map_err(|e| format!("Cannot read {path}: {e}"))?; + result.steps.push(SchemaUpgradeStep { path: path.to_owned(), sql }); } Ok(result) } @@ -187,7 +197,8 @@ impl DataStore { ) .map_err(|e| format!("Invalid schema path: {}", e.display()))?; - let up_sqls = all_sql_for_version_migration(&target_dir).await?; + let schema_change = + all_sql_for_version_migration(&target_dir).await?; // Confirm the current version, set the "target_version" // column to indicate that a schema update is in-progress. @@ -205,7 +216,7 @@ impl DataStore { "target_version" => target_version.to_string(), ); - for sql in &up_sqls { + for SchemaUpgradeStep { path: _, sql } in &schema_change.steps { // Perform the schema change. self.apply_schema_update( ¤t_version, diff --git a/nexus/db-queries/src/db/datastore/image.rs b/nexus/db-queries/src/db/datastore/image.rs index e44da013cd..759b523010 100644 --- a/nexus/db-queries/src/db/datastore/image.rs +++ b/nexus/db-queries/src/db/datastore/image.rs @@ -19,6 +19,7 @@ use nexus_db_model::Name; use nexus_types::identity::Resource; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; use omicron_common::api::external::UpdateResult; @@ -232,4 +233,40 @@ impl DataStore { })?; Ok(image) } + + pub async fn silo_image_delete( + &self, + opctx: &OpContext, + authz_image: &authz::SiloImage, + image: SiloImage, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_image).await?; + self.image_delete(opctx, image.into()).await + } + + pub async fn project_image_delete( + &self, + opctx: &OpContext, + authz_image: &authz::ProjectImage, + image: ProjectImage, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_image).await?; + self.image_delete(opctx, image.into()).await + } + + async fn image_delete( + &self, + opctx: &OpContext, + image: Image, + ) -> DeleteResult { + use db::schema::image::dsl; + diesel::update(dsl::image) + .filter(dsl::id.eq(image.id())) + .set(dsl::time_deleted.eq(Utc::now())) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } } diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 3b2d81e1c7..91373f6875 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -48,6 +48,8 @@ use std::sync::Arc; use uuid::Uuid; mod address_lot; +mod bgp; +mod bootstore; mod certificate; mod console_session; mod dataset; @@ -90,7 +92,8 @@ mod zpool; pub use address_lot::AddressLotCreateResult; pub use db_metadata::{ - all_sql_for_version_migration, EARLIEST_SUPPORTED_VERSION, + all_sql_for_version_migration, SchemaUpgrade, SchemaUpgradeStep, + EARLIEST_SUPPORTED_VERSION, }; pub use dns::DnsVersionUpdateBuilder; pub use instance::InstanceAndActiveVmm; diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index f5f7524aab..ae982d86f8 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -32,6 +32,7 @@ use chrono::Utc; use diesel::prelude::*; use diesel::result::Error as DieselError; use diesel::upsert::excluded; +use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::DnsZone; use nexus_db_model::ExternalIp; @@ -61,6 +62,7 @@ use uuid::Uuid; #[derive(Clone)] pub struct RackInit { pub rack_id: Uuid, + pub rack_subnet: IpNetwork, pub services: Vec, pub datasets: Vec, pub service_ip_pool_ranges: Vec, @@ -190,6 +192,28 @@ impl DataStore { }) } + pub async fn update_rack_subnet( + &self, + opctx: &OpContext, + rack: &Rack, + ) -> Result<(), Error> { + debug!( + opctx.log, + "updating rack subnet for rack {} to {:#?}", + rack.id(), + rack.rack_subnet + ); + use db::schema::rack::dsl; + diesel::update(dsl::rack) + .filter(dsl::id.eq(rack.id())) + .set(dsl::rack_subnet.eq(rack.rack_subnet)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } + // The following methods which return a `TxnError` take a `conn` parameter // which comes from the transaction created in `rack_set_initialized`. @@ -681,6 +705,7 @@ mod test { fn default() -> Self { RackInit { rack_id: Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap(), + rack_subnet: nexus_test_utils::RACK_SUBNET.parse().unwrap(), services: vec![], datasets: vec![], service_ip_pool_ranges: vec![], diff --git a/nexus/db-queries/src/db/datastore/snapshot.rs b/nexus/db-queries/src/db/datastore/snapshot.rs index 59fb00c84d..7c03e4bd40 100644 --- a/nexus/db-queries/src/db/datastore/snapshot.rs +++ b/nexus/db-queries/src/db/datastore/snapshot.rs @@ -19,6 +19,7 @@ use crate::db::model::Snapshot; use crate::db::model::SnapshotState; use crate::db::pagination::paginated; use crate::db::update_and_check::UpdateAndCheck; +use crate::db::update_and_check::UpdateStatus; use crate::db::TransactionError; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; @@ -252,30 +253,70 @@ impl DataStore { use db::schema::snapshot::dsl; - let updated_rows = diesel::update(dsl::snapshot) + let result = diesel::update(dsl::snapshot) .filter(dsl::time_deleted.is_null()) .filter(dsl::gen.eq(gen)) .filter(dsl::id.eq(snapshot_id)) - .filter(dsl::state.eq_any(ok_to_delete_states)) + .filter(dsl::state.eq_any(ok_to_delete_states.clone())) .set(( dsl::time_deleted.eq(now), dsl::state.eq(SnapshotState::Destroyed), )) .check_if_exists::(snapshot_id) - .execute_async(&*self.pool_connection_authorized(&opctx).await?) + .execute_and_check(&*self.pool_connection_authorized(&opctx).await?) .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Snapshot, + LookupType::ById(snapshot_id), + ), + ) + })?; + + match result.status { + UpdateStatus::Updated => { + // snapshot was soft deleted ok + Ok(result.found.id()) + } - if updated_rows == 0 { - // Either: - // - // - the snapshot was already deleted - // - the generation number changed - // - the state of the snapshot isn't one of `ok_to_delete_states` + UpdateStatus::NotUpdatedButExists => { + let snapshot = result.found; - return Err(Error::invalid_request("snapshot cannot be deleted")); - } + // if the snapshot was already deleted, return Ok - this + // function must remain idempotent for the same input. + if snapshot.time_deleted().is_some() + && snapshot.state == SnapshotState::Destroyed + { + Ok(snapshot.id()) + } else { + // if the snapshot was not deleted, figure out why + if !ok_to_delete_states.contains(&snapshot.state) { + Err(Error::invalid_request(&format!( + "snapshot cannot be deleted in state {:?}", + snapshot.state, + ))) + } else if snapshot.gen != gen { + Err(Error::invalid_request(&format!( + "snapshot cannot be deleted: mismatched generation {:?} != {:?}", + gen, + snapshot.gen, + ))) + } else { + error!( + opctx.log, + "snapshot exists but cannot be deleted: {:?} (db_snapshot is {:?}", + snapshot, + db_snapshot, + ); - Ok(snapshot_id) + Err(Error::invalid_request( + "snapshot exists but cannot be deleted", + )) + } + } + } + } } } diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index 45be594be6..f301750ee9 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -97,43 +97,86 @@ pub struct SwitchPortSettingsGroupCreateResult { } impl DataStore { - // port settings + pub async fn switch_port_settings_exist( + &self, + opctx: &OpContext, + name: Name, + ) -> LookupResult { + use db::schema::switch_port_settings::{ + self, dsl as port_settings_dsl, + }; + + let pool = self.pool_connection_authorized(opctx).await?; + + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::name.eq(name)) + .select(switch_port_settings::id) + .limit(1) + .first_async::(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn switch_ports_using_settings( + &self, + opctx: &OpContext, + switch_port_settings_id: Uuid, + ) -> LookupResult> { + use db::schema::switch_port::{self, dsl}; + + let pool = self.pool_connection_authorized(opctx).await?; + + dsl::switch_port + .filter(switch_port::port_settings_id.eq(switch_port_settings_id)) + .select((switch_port::id, switch_port::port_name)) + .load_async(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } pub async fn switch_port_settings_create( &self, opctx: &OpContext, params: ¶ms::SwitchPortSettingsCreate, + id: Option, ) -> CreateResult { - use db::schema::address_lot::dsl as address_lot_dsl; - use db::schema::bgp_announce_set::dsl as bgp_announce_set_dsl; - use db::schema::bgp_config::dsl as bgp_config_dsl; - use db::schema::lldp_service_config::dsl as lldp_config_dsl; - use db::schema::switch_port_settings::dsl as port_settings_dsl; - use db::schema::switch_port_settings_address_config::dsl as address_config_dsl; - use db::schema::switch_port_settings_bgp_peer_config::dsl as bgp_peer_dsl; - use db::schema::switch_port_settings_interface_config::dsl as interface_config_dsl; - use db::schema::switch_port_settings_link_config::dsl as link_config_dsl; - use db::schema::switch_port_settings_port_config::dsl as port_config_dsl; - use db::schema::switch_port_settings_route_config::dsl as route_config_dsl; - use db::schema::switch_vlan_interface_config::dsl as vlan_config_dsl; + use db::schema::{ + address_lot::dsl as address_lot_dsl, + //XXX ANNOUNCE bgp_announce_set::dsl as bgp_announce_set_dsl, + bgp_config::dsl as bgp_config_dsl, + lldp_service_config::dsl as lldp_config_dsl, + switch_port_settings::dsl as port_settings_dsl, + switch_port_settings_address_config::dsl as address_config_dsl, + switch_port_settings_bgp_peer_config::dsl as bgp_peer_dsl, + switch_port_settings_interface_config::dsl as interface_config_dsl, + switch_port_settings_link_config::dsl as link_config_dsl, + switch_port_settings_port_config::dsl as port_config_dsl, + switch_port_settings_route_config::dsl as route_config_dsl, + switch_vlan_interface_config::dsl as vlan_config_dsl, + }; #[derive(Debug)] enum SwitchPortSettingsCreateError { AddressLotNotFound, - BgpAnnounceSetNotFound, + //XXX ANNOUNCE BgpAnnounceSetNotFound, BgpConfigNotFound, ReserveBlock(ReserveBlockError), } type TxnError = TransactionError; + type SpsCreateError = SwitchPortSettingsCreateError; let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage conn.transaction_async(|conn| async move { - // create the top level port settings object - let port_settings = SwitchPortSettings::new(¶ms.identity); + let port_settings = match id { + Some(id) => SwitchPortSettings::with_id(id, ¶ms.identity), + None => SwitchPortSettings::new(¶ms.identity), + }; + //let port_settings = SwitchPortSettings::new(¶ms.identity); let db_port_settings: SwitchPortSettings = diesel::insert_into(port_settings_dsl::switch_port_settings) .values(port_settings) @@ -189,6 +232,8 @@ impl DataStore { lldp_svc_config.id, link_name.clone(), c.mtu, + c.fec.into(), + c.speed.into(), )); } result.link_lldp = @@ -260,33 +305,6 @@ impl DataStore { let mut bgp_peer_config = Vec::new(); for (interface_name, p) in ¶ms.bgp_peers { - - // add the bgp peer - // TODO this requires pluming in the API to create - // - bgp configs - // - announce sets - // - announcements - - use db::schema::bgp_announce_set; - let announce_set_id = match &p.bgp_announce_set { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - bgp_announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::time_deleted.is_null()) - .filter(bgp_announce_set::name.eq(name)) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpAnnounceSetNotFound, - ) - })? - } - }; - use db::schema::bgp_config; let bgp_config_id = match &p.bgp_config { NameOrId::Id(id) => *id, @@ -309,10 +327,14 @@ impl DataStore { bgp_peer_config.push(SwitchPortBgpPeerConfig::new( psid, - announce_set_id, bgp_config_id, interface_name.clone(), p.addr.into(), + p.hold_time.into(), + p.idle_hold_time.into(), + p.delay_open.into(), + p.connect_retry.into(), + p.keepalive.into(), )); } @@ -389,16 +411,10 @@ impl DataStore { }) .await .map_err(|e| match e { - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpAnnounceSetNotFound) => { - Error::invalid_request("BGP announce set not found") - } - TxnError::CustomError( - SwitchPortSettingsCreateError::AddressLotNotFound) => { + TxnError::CustomError(SpsCreateError::AddressLotNotFound) => { Error::invalid_request("AddressLot not found") } - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpConfigNotFound) => { + TxnError::CustomError(SpsCreateError::BgpConfigNotFound) => { Error::invalid_request("BGP config not found") } TxnError::CustomError( @@ -475,30 +491,31 @@ impl DataStore { .await?; // delete the port config object - use db::schema::switch_port_settings_port_config; - use db::schema::switch_port_settings_port_config::dsl as port_config_dsl; + use db::schema::switch_port_settings_port_config::{ + self as sps_port_config, dsl as port_config_dsl, + }; diesel::delete(port_config_dsl::switch_port_settings_port_config) - .filter(switch_port_settings_port_config::port_settings_id.eq(id)) + .filter(sps_port_config::port_settings_id.eq(id)) .execute_async(&conn) .await?; // delete the link configs - use db::schema::switch_port_settings_link_config; - use db::schema::switch_port_settings_link_config::dsl as link_config_dsl; + use db::schema::switch_port_settings_link_config::{ + self as sps_link_config, dsl as link_config_dsl, + }; let links: Vec = diesel::delete( link_config_dsl::switch_port_settings_link_config ) .filter( - switch_port_settings_link_config::port_settings_id.eq(id) + sps_link_config::port_settings_id.eq(id) ) .returning(SwitchPortLinkConfig::as_returning()) .get_results_async(&conn) .await?; // delete lldp configs - use db::schema::lldp_service_config; - use db::schema::lldp_service_config::dsl as lldp_config_dsl; + use db::schema::lldp_service_config::{self, dsl as lldp_config_dsl}; let lldp_svc_ids: Vec = links .iter() .map(|link| link.lldp_service_config_id) @@ -509,26 +526,25 @@ impl DataStore { .await?; // delete interface configs - use db::schema::switch_port_settings_interface_config; - use db::schema::switch_port_settings_interface_config::dsl - as interface_config_dsl; + use db::schema::switch_port_settings_interface_config::{ + self as sps_interface_config, dsl as interface_config_dsl, + }; let interfaces: Vec = diesel::delete( interface_config_dsl::switch_port_settings_interface_config ) .filter( - switch_port_settings_interface_config::port_settings_id.eq( - id - ) + sps_interface_config::port_settings_id.eq(id) ) .returning(SwitchInterfaceConfig::as_returning()) .get_results_async(&conn) .await?; // delete any vlan interfaces - use db::schema::switch_vlan_interface_config; - use db::schema::switch_vlan_interface_config::dsl as vlan_config_dsl; + use db::schema::switch_vlan_interface_config::{ + self, dsl as vlan_config_dsl, + }; let interface_ids: Vec = interfaces .iter() .map(|interface| interface.id) @@ -566,22 +582,26 @@ impl DataStore { .await?; // delete address configs - use db::schema::switch_port_settings_address_config as address_config; - use db::schema::switch_port_settings_address_config::dsl - as address_config_dsl; + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; - let ps = diesel::delete(address_config_dsl::switch_port_settings_address_config) - .filter(address_config::port_settings_id.eq(id)) - .returning(SwitchPortAddressConfig::as_returning()) - .get_result_async(&conn) - .await?; + let port_settings_addrs = diesel::delete( + address_config_dsl::switch_port_settings_address_config, + ) + .filter(address_config::port_settings_id.eq(id)) + .returning(SwitchPortAddressConfig::as_returning()) + .get_results_async(&conn) + .await?; use db::schema::address_lot_rsvd_block::dsl as rsvd_block_dsl; - diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) - .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) - .execute_async(&conn) - .await?; + for ps in &port_settings_addrs { + diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) + .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) + .execute_async(&conn) + .await?; + } Ok(()) }) @@ -650,10 +670,10 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage conn.transaction_async(|conn| async move { - // get the top level port settings object - use db::schema::switch_port_settings::dsl as port_settings_dsl; - use db::schema::switch_port_settings; + use db::schema::switch_port_settings::{ + self, dsl as port_settings_dsl, + }; let id = match name_or_id { NameOrId::Id(id) => *id, @@ -668,23 +688,27 @@ impl DataStore { .await .map_err(|_| { TxnError::CustomError( - SwitchPortSettingsGetError::NotFound(name.clone()) + SwitchPortSettingsGetError::NotFound( + name.clone(), + ), ) })? } }; - let settings: SwitchPortSettings = port_settings_dsl::switch_port_settings - .filter(switch_port_settings::time_deleted.is_null()) - .filter(switch_port_settings::id.eq(id)) - .select(SwitchPortSettings::as_select()) - .limit(1) - .first_async::(&conn) - .await?; + let settings: SwitchPortSettings = + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::id.eq(id)) + .select(SwitchPortSettings::as_select()) + .limit(1) + .first_async::(&conn) + .await?; // get the port config - use db::schema::switch_port_settings_port_config::dsl as port_config_dsl; - use db::schema::switch_port_settings_port_config as port_config; + use db::schema::switch_port_settings_port_config::{ + self as port_config, dsl as port_config_dsl, + }; let port: SwitchPortConfig = port_config_dsl::switch_port_settings_port_config .filter(port_config::port_settings_id.eq(id)) @@ -694,11 +718,13 @@ impl DataStore { .await?; // initialize result - let mut result = SwitchPortSettingsCombinedResult::new(settings, port); + let mut result = + SwitchPortSettingsCombinedResult::new(settings, port); // get the link configs - use db::schema::switch_port_settings_link_config::dsl as link_config_dsl; - use db::schema::switch_port_settings_link_config as link_config; + use db::schema::switch_port_settings_link_config::{ + self as link_config, dsl as link_config_dsl, + }; result.links = link_config_dsl::switch_port_settings_link_config .filter(link_config::port_settings_id.eq(id)) @@ -706,25 +732,25 @@ impl DataStore { .load_async::(&conn) .await?; - let lldp_svc_ids: Vec = result.links + let lldp_svc_ids: Vec = result + .links .iter() .map(|link| link.lldp_service_config_id) .collect(); - use db::schema::lldp_service_config::dsl as lldp_dsl; use db::schema::lldp_service_config as lldp_config; - result.link_lldp = - lldp_dsl::lldp_service_config - .filter(lldp_config::id.eq_any(lldp_svc_ids)) - .select(LldpServiceConfig::as_select()) - .limit(1) - .load_async::(&conn) - .await?; + use db::schema::lldp_service_config::dsl as lldp_dsl; + result.link_lldp = lldp_dsl::lldp_service_config + .filter(lldp_config::id.eq_any(lldp_svc_ids)) + .select(LldpServiceConfig::as_select()) + .limit(1) + .load_async::(&conn) + .await?; // get the interface configs - use db::schema::switch_port_settings_interface_config::dsl - as interface_config_dsl; - use db::schema::switch_port_settings_interface_config as interface_config; + use db::schema::switch_port_settings_interface_config::{ + self as interface_config, dsl as interface_config_dsl, + }; result.interfaces = interface_config_dsl::switch_port_settings_interface_config @@ -733,37 +759,35 @@ impl DataStore { .load_async::(&conn) .await?; - use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; use db::schema::switch_vlan_interface_config as vlan_config; - let interface_ids: Vec = result.interfaces + use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; + let interface_ids: Vec = result + .interfaces .iter() .map(|interface| interface.id) .collect(); - result.vlan_interfaces = - vlan_dsl::switch_vlan_interface_config - .filter( - vlan_config::interface_config_id.eq_any(interface_ids) - ) + result.vlan_interfaces = vlan_dsl::switch_vlan_interface_config + .filter(vlan_config::interface_config_id.eq_any(interface_ids)) .select(SwitchVlanInterfaceConfig::as_select()) .load_async::(&conn) .await?; - // get the route configs - use db::schema::switch_port_settings_route_config::dsl as route_config_dsl; - use db::schema::switch_port_settings_route_config as route_config; + use db::schema::switch_port_settings_route_config::{ + self as route_config, dsl as route_config_dsl, + }; - result.routes = - route_config_dsl::switch_port_settings_route_config - .filter(route_config::port_settings_id.eq(id)) - .select(SwitchPortRouteConfig::as_select()) - .load_async::(&conn) - .await?; + result.routes = route_config_dsl::switch_port_settings_route_config + .filter(route_config::port_settings_id.eq(id)) + .select(SwitchPortRouteConfig::as_select()) + .load_async::(&conn) + .await?; // get the bgp peer configs - use db::schema::switch_port_settings_bgp_peer_config::dsl as bgp_peer_dsl; - use db::schema::switch_port_settings_bgp_peer_config as bgp_peer; + use db::schema::switch_port_settings_bgp_peer_config::{ + self as bgp_peer, dsl as bgp_peer_dsl, + }; result.bgp_peers = bgp_peer_dsl::switch_port_settings_bgp_peer_config @@ -773,9 +797,9 @@ impl DataStore { .await?; // get the address configs - use db::schema::switch_port_settings_address_config::dsl - as address_config_dsl; - use db::schema::switch_port_settings_address_config as address_config; + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; result.addresses = address_config_dsl::switch_port_settings_address_config @@ -785,14 +809,15 @@ impl DataStore { .await?; Ok(result) - }) .await .map_err(|e| match e { - TxnError::CustomError( - SwitchPortSettingsGetError::NotFound(name)) => { - Error::not_found_by_name(ResourceType::SwitchPortSettings, &name) - } + TxnError::CustomError(SwitchPortSettingsGetError::NotFound( + name, + )) => Error::not_found_by_name( + ResourceType::SwitchPortSettings, + &name, + ), TxnError::Database(e) => match e { DieselError::DatabaseError(_, _) => { let name = name_or_id.to_string(); @@ -803,7 +828,7 @@ impl DataStore { &name, ), ) - }, + } _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) @@ -1083,8 +1108,10 @@ impl DataStore { &self, opctx: &OpContext, ) -> ListResultVec { - use db::schema::switch_port::dsl as switch_port_dsl; - use db::schema::switch_port_settings_route_config::dsl as route_config_dsl; + use db::schema::{ + switch_port::dsl as switch_port_dsl, + switch_port_settings_route_config::dsl as route_config_dsl, + }; switch_port_dsl::switch_port .filter(switch_port_dsl::port_settings_id.is_not_null()) diff --git a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs index 18ff58735e..83856e10c7 100644 --- a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs +++ b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs @@ -272,7 +272,11 @@ impl DataStore { Ok(provisions) } - /// Transitively updates all CPU/RAM provisions from project -> fleet. + /// Transitively removes the CPU and memory charges for an instance from the + /// instance's project, silo, and fleet, provided that the instance's state + /// generation is less than `max_instance_gen`. This allows a caller who is + /// about to apply generation G to an instance to avoid deleting resources + /// if its update was superseded. pub async fn virtual_provisioning_collection_delete_instance( &self, opctx: &OpContext, @@ -280,10 +284,15 @@ impl DataStore { project_id: Uuid, cpus_diff: i64, ram_diff: ByteCount, + max_instance_gen: i64, ) -> Result, Error> { let provisions = VirtualProvisioningCollectionUpdate::new_delete_instance( - id, cpus_diff, ram_diff, project_id, + id, + max_instance_gen, + cpus_diff, + ram_diff, + project_id, ) .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index 38e3875036..5d753f0742 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -14,9 +14,10 @@ use crate::db::model::Dataset; use crate::db::model::Region; use crate::db::model::RegionSnapshot; use crate::db::model::Volume; +use crate::db::queries::volume::DecreaseCrucibleResourceCountAndSoftDeleteVolume; +use anyhow::bail; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; -use chrono::Utc; use diesel::prelude::*; use diesel::OptionalExtension; use omicron_common::api::external::CreateResult; @@ -26,6 +27,7 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::ResourceType; use serde::Deserialize; +use serde::Deserializer; use serde::Serialize; use sled_agent_client::types::VolumeConstructionRequest; use uuid::Uuid; @@ -391,6 +393,16 @@ impl DataStore { opts, gen, } => { + if !opts.read_only { + // Only one volume can "own" a Region, and that volume's + // UUID is recorded in the region table accordingly. It is + // an error to make a copy of a volume construction request + // that references non-read-only Regions. + bail!( + "only one Volume can reference a Region non-read-only!" + ); + } + let mut opts = opts.clone(); opts.id = Uuid::new_v4(); @@ -416,7 +428,10 @@ impl DataStore { /// Checkout a copy of the Volume from the database using `volume_checkout`, /// then randomize the UUIDs in the construction request. Because this is a /// new volume, it is immediately passed to `volume_create` so that the - /// accounting for Crucible resources stays correct. + /// accounting for Crucible resources stays correct. This is only valid for + /// Volumes that reference regions read-only - it's important for accounting + /// purposes that each region in this volume construction request is + /// returned by `read_only_resources_associated_with_volume`. pub async fn volume_checkout_randomize_ids( &self, volume_id: Uuid, @@ -469,6 +484,8 @@ impl DataStore { .eq(0) // Despite the SQL specifying that this column is NOT NULL, // this null check is required for this function to work! + // It's possible that the left join of region_snapshot above + // could join zero rows, making this null. .or(dsl::volume_references.is_null()), ) // where the volume has already been soft-deleted @@ -517,16 +534,6 @@ impl DataStore { &self, volume_id: Uuid, ) -> Result { - #[derive(Debug, thiserror::Error)] - enum DecreaseCrucibleResourcesError { - #[error("Error during decrease Crucible resources: {0}")] - DieselError(#[from] diesel::result::Error), - - #[error("Serde error during decrease Crucible resources: {0}")] - SerdeError(#[from] serde_json::Error), - } - type TxnError = TransactionError; - // Grab all the targets that the volume construction request references. // Do this outside the transaction, as the data inside volume doesn't // change and this would simply add to the transaction time. @@ -558,12 +565,13 @@ impl DataStore { crucible_targets }; - // In a transaction: + // Call a CTE that will: // // 1. decrease the number of references for each region snapshot that // this Volume references // 2. soft-delete the volume - // 3. record the resources to clean up + // 3. record the resources to clean up as a serialized CrucibleResources + // struct in volume's `resources_to_clean_up` column. // // Step 3 is important because this function is called from a saga node. // If saga execution crashes after steps 1 and 2, but before serializing @@ -572,197 +580,48 @@ impl DataStore { // // We also have to guard against the case where this function is called // multiple times, and that is done by soft-deleting the volume during - // the transaction, and returning the previously serialized list of - // resources to clean up if a soft-delete has already occurred. - - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - // Grab the volume in question. If the volume record was already - // hard-deleted, assume clean-up has occurred and return an empty - // CrucibleResources. If the volume record was soft-deleted, then - // return the serialized CrucibleResources. - use db::schema::volume::dsl as volume_dsl; - - { - let volume = volume_dsl::volume - .filter(volume_dsl::id.eq(volume_id)) - .select(Volume::as_select()) - .get_result_async(&conn) - .await - .optional()?; + // the CTE, and returning the previously serialized list of resources to + // clean up if a soft-delete has already occurred. - let volume = if let Some(v) = volume { - v - } else { - // the volume was hard-deleted, return an empty - // CrucibleResources - return Ok(CrucibleResources::V1( - CrucibleResourcesV1::default(), - )); - }; + let _old_volume: Vec = + DecreaseCrucibleResourceCountAndSoftDeleteVolume::new( + volume_id, + crucible_targets.read_only_targets.clone(), + ) + .get_results_async::( + &*self.pool_connection_unauthorized().await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; - if volume.time_deleted.is_none() { - // a volume record exists, and was not deleted - this is the - // first time through this transaction for a certain volume - // id. Get the volume for use in the transaction. - volume - } else { - // this volume was soft deleted - this is a repeat time - // through this transaction. - - if let Some(resources_to_clean_up) = - volume.resources_to_clean_up - { - // return the serialized CrucibleResources - return serde_json::from_str( - &resources_to_clean_up, - ) - .map_err(|e| { - TxnError::CustomError( - DecreaseCrucibleResourcesError::SerdeError( - e, - ), - ) - }); - } else { - // If no CrucibleResources struct was serialized, that's - // definitely a bug of some sort - the soft-delete below - // sets time_deleted at the same time as - // resources_to_clean_up! But, instead of a panic here, - // just return an empty CrucibleResources. - return Ok(CrucibleResources::V1( - CrucibleResourcesV1::default(), - )); - } + // Get the updated Volume to get the resources to clean up + let resources_to_clean_up: CrucibleResources = match self + .volume_get(volume_id) + .await? + { + Some(volume) => { + match volume.resources_to_clean_up.as_ref() { + Some(v) => serde_json::from_str(v)?, + + None => { + // Even volumes with nothing to clean up should have + // a serialized CrucibleResources that contains + // empty vectors instead of None. Instead of + // panicing here though, just return the default + // (nothing to clean up). + CrucibleResources::V1(CrucibleResourcesV1::default()) } - }; - - // Decrease the number of uses for each non-deleted referenced - // region snapshot. - - use db::schema::region_snapshot::dsl; - - diesel::update(dsl::region_snapshot) - .filter( - dsl::snapshot_addr - .eq_any(crucible_targets.read_only_targets.clone()), - ) - .filter(dsl::volume_references.gt(0)) - .filter(dsl::deleting.eq(false)) - .set(dsl::volume_references.eq(dsl::volume_references - 1)) - .execute_async(&conn) - .await?; - - // Then, note anything that was set to zero from the above - // UPDATE, and then mark all those as deleted. - let snapshots_to_delete: Vec = - dsl::region_snapshot - .filter( - dsl::snapshot_addr.eq_any( - crucible_targets.read_only_targets.clone(), - ), - ) - .filter(dsl::volume_references.eq(0)) - .filter(dsl::deleting.eq(false)) - .select(RegionSnapshot::as_select()) - .load_async(&conn) - .await?; - - diesel::update(dsl::region_snapshot) - .filter( - dsl::snapshot_addr - .eq_any(crucible_targets.read_only_targets.clone()), - ) - .filter(dsl::volume_references.eq(0)) - .filter(dsl::deleting.eq(false)) - .set(dsl::deleting.eq(true)) - .execute_async(&conn) - .await?; - - // Return what results can be cleaned up - let result = CrucibleResources::V2(CrucibleResourcesV2 { - // The only use of a read-write region will be at the top level of a - // Volume. These are not shared, but if any snapshots are taken this - // will prevent deletion of the region. Filter out any regions that - // have associated snapshots. - datasets_and_regions: { - use db::schema::dataset::dsl as dataset_dsl; - use db::schema::region::dsl as region_dsl; - - // Return all regions for this volume_id, where there either are - // no region_snapshots, or region_snapshots.volume_references = - // 0. - region_dsl::region - .filter(region_dsl::volume_id.eq(volume_id)) - .inner_join( - dataset_dsl::dataset - .on(region_dsl::dataset_id - .eq(dataset_dsl::id)), - ) - .left_join( - dsl::region_snapshot.on(dsl::region_id - .eq(region_dsl::id) - .and(dsl::dataset_id.eq(dataset_dsl::id))), - ) - .filter( - dsl::volume_references - .eq(0) - // Despite the SQL specifying that this column is NOT NULL, - // this null check is required for this function to work! - // The left join of region_snapshot might cause a null here. - .or(dsl::volume_references.is_null()), - ) - .select((Dataset::as_select(), Region::as_select())) - .get_results_async::<(Dataset, Region)>(&conn) - .await? - }, - - // Consumers of this struct will be responsible for deleting - // the read-only downstairs running for the snapshot and the - // snapshot itself. - // - // It's important to not return *every* region snapshot with - // zero references: multiple volume delete sub-sagas will - // then be issues duplicate DELETE calls to Crucible agents, - // and a request to delete a read-only downstairs running - // for a snapshot that doesn't exist will return a 404, - // causing the saga to error and unwind. - snapshots_to_delete, - }); - - // Soft delete this volume, and serialize the resources that are to - // be cleaned up. - let now = Utc::now(); - diesel::update(volume_dsl::volume) - .filter(volume_dsl::id.eq(volume_id)) - .set(( - volume_dsl::time_deleted.eq(now), - volume_dsl::resources_to_clean_up.eq( - serde_json::to_string(&result).map_err(|e| { - TxnError::CustomError( - DecreaseCrucibleResourcesError::SerdeError( - e, - ), - ) - })?, - ), - )) - .execute_async(&conn) - .await?; + } + } - Ok(result) - }) - .await - .map_err(|e| match e { - TxnError::CustomError( - DecreaseCrucibleResourcesError::DieselError(e), - ) => public_error_from_diesel(e, ErrorHandler::Server), + None => { + // If the volume was hard-deleted already, return the + // default (nothing to clean up). + CrucibleResources::V1(CrucibleResourcesV1::default()) + } + }; - _ => { - Error::internal_error(&format!("Transaction error: {}", e)) - } - }) + Ok(resources_to_clean_up) } // Here we remove the read only parent from volume_id, and attach it @@ -977,6 +836,7 @@ pub struct CrucibleTargets { pub enum CrucibleResources { V1(CrucibleResourcesV1), V2(CrucibleResourcesV2), + V3(CrucibleResourcesV3), } #[derive(Debug, Default, Serialize, Deserialize)] @@ -991,6 +851,176 @@ pub struct CrucibleResourcesV2 { pub snapshots_to_delete: Vec, } +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct RegionSnapshotV3 { + dataset: Uuid, + region: Uuid, + snapshot: Uuid, +} + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct CrucibleResourcesV3 { + #[serde(deserialize_with = "null_to_empty_list")] + pub regions: Vec, + + #[serde(deserialize_with = "null_to_empty_list")] + pub region_snapshots: Vec, +} + +// Cockroach's `json_agg` will emit a `null` instead of a `[]` if a SELECT +// returns zero rows. Handle that with this function when deserializing. +fn null_to_empty_list<'de, D, T>(de: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + Ok(match Option::>::deserialize(de)? { + Some(v) => v, + None => vec![], + }) +} + +impl DataStore { + /// For a CrucibleResources object, return the Regions to delete, as well as + /// the Dataset they belong to. + pub async fn regions_to_delete( + &self, + crucible_resources: &CrucibleResources, + ) -> LookupResult> { + let conn = self.pool_connection_unauthorized().await?; + + match crucible_resources { + CrucibleResources::V1(crucible_resources) => { + Ok(crucible_resources.datasets_and_regions.clone()) + } + + CrucibleResources::V2(crucible_resources) => { + Ok(crucible_resources.datasets_and_regions.clone()) + } + + CrucibleResources::V3(crucible_resources) => { + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region::dsl as region_dsl; + + region_dsl::region + .filter( + region_dsl::id + .eq_any(crucible_resources.regions.clone()), + ) + .inner_join( + dataset_dsl::dataset + .on(region_dsl::dataset_id.eq(dataset_dsl::id)), + ) + .select((Dataset::as_select(), Region::as_select())) + .get_results_async::<(Dataset, Region)>(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + }) + } + } + } + + /// For a CrucibleResources object, return the RegionSnapshots to delete, as + /// well as the Dataset they belong to. + pub async fn snapshots_to_delete( + &self, + crucible_resources: &CrucibleResources, + ) -> LookupResult> { + let conn = self.pool_connection_unauthorized().await?; + + match crucible_resources { + CrucibleResources::V1(crucible_resources) => { + Ok(crucible_resources.datasets_and_snapshots.clone()) + } + + CrucibleResources::V2(crucible_resources) => { + use db::schema::dataset::dsl; + + let mut result: Vec<_> = Vec::with_capacity( + crucible_resources.snapshots_to_delete.len(), + ); + + for snapshots_to_delete in + &crucible_resources.snapshots_to_delete + { + let maybe_dataset = dsl::dataset + .filter(dsl::id.eq(snapshots_to_delete.dataset_id)) + .select(Dataset::as_select()) + .first_async(&*conn) + .await + .optional() + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + match maybe_dataset { + Some(dataset) => { + result.push((dataset, snapshots_to_delete.clone())); + } + + None => { + return Err(Error::internal_error(&format!( + "could not find dataset {}!", + snapshots_to_delete.dataset_id, + ))); + } + } + } + + Ok(result) + } + + CrucibleResources::V3(crucible_resources) => { + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region_snapshot::dsl; + + let mut datasets_and_snapshots = Vec::with_capacity( + crucible_resources.region_snapshots.len(), + ); + + for region_snapshots in &crucible_resources.region_snapshots { + let maybe_tuple = dsl::region_snapshot + .filter(dsl::dataset_id.eq(region_snapshots.dataset)) + .filter(dsl::region_id.eq(region_snapshots.region)) + .filter(dsl::snapshot_id.eq(region_snapshots.snapshot)) + .inner_join( + dataset_dsl::dataset + .on(dsl::dataset_id.eq(dataset_dsl::id)), + ) + .select(( + Dataset::as_select(), + RegionSnapshot::as_select(), + )) + .first_async::<(Dataset, RegionSnapshot)>(&*conn) + .await + .optional() + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + match maybe_tuple { + Some(tuple) => { + datasets_and_snapshots.push(tuple); + } + + None => { + // If something else is deleting the exact same + // CrucibleResources (for example from a duplicate + // resource-delete saga) then these region_snapshot + // entries could be gone (because they are hard + // deleted). Skip missing entries, return only what + // we can find. + } + } + } + + Ok(datasets_and_snapshots) + } + } + } +} + /// Return the targets from a VolumeConstructionRequest. /// /// The targets of a volume construction request map to resources. diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 14886ba018..6db99465a3 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -34,12 +34,15 @@ use crate::db::model::VpcUpdate; use crate::db::model::{Ipv4Net, Ipv6Net}; use crate::db::pagination::paginated; use crate::db::queries::vpc::InsertVpcQuery; +use crate::db::queries::vpc::VniSearchIter; use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery; use crate::db::queries::vpc_subnet::SubnetError; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; +use diesel::result::DatabaseErrorKind; +use diesel::result::Error as DieselError; use ipnetwork::IpNetwork; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; @@ -85,18 +88,23 @@ impl DataStore { SERVICES_VPC.clone(), Some(Vni(ExternalVni::SERVICES_VNI)), ); - let authz_vpc = self + let authz_vpc = match self .project_create_vpc_raw(opctx, &authz_project, vpc_query) .await - .map(|(authz_vpc, _)| authz_vpc) - .or_else(|e| match e { - Error::ObjectAlreadyExists { .. } => Ok(authz::Vpc::new( - authz_project.clone(), - *SERVICES_VPC_ID, - LookupType::ByName(SERVICES_VPC.identity.name.to_string()), - )), - _ => Err(e), - })?; + { + Ok(None) => { + let msg = "VNI exhaustion detected when creating built-in VPCs"; + error!(opctx.log, "{}", msg); + Err(Error::internal_error(msg)) + } + Ok(Some((authz_vpc, _))) => Ok(authz_vpc), + Err(Error::ObjectAlreadyExists { .. }) => Ok(authz::Vpc::new( + authz_project.clone(), + *SERVICES_VPC_ID, + LookupType::ByName(SERVICES_VPC.identity.name.to_string()), + )), + Err(e) => Err(e), + }?; // Also add the system router and internet gateway route @@ -287,22 +295,65 @@ impl DataStore { &self, opctx: &OpContext, authz_project: &authz::Project, - vpc: IncompleteVpc, + mut vpc: IncompleteVpc, ) -> Result<(authz::Vpc, Vpc), Error> { - self.project_create_vpc_raw( - opctx, - authz_project, - InsertVpcQuery::new(vpc), - ) - .await + // Generate an iterator that allows us to search the entire space of + // VNIs for this VPC, in manageable chunks to limit memory usage. + let vnis = VniSearchIter::new(vpc.vni.0); + for (i, vni) in vnis.enumerate() { + vpc.vni = Vni(vni); + let id = usdt::UniqueId::new(); + crate::probes::vni__search__range__start!(|| { + (&id, u32::from(vni), VniSearchIter::STEP_SIZE) + }); + match self + .project_create_vpc_raw( + opctx, + authz_project, + InsertVpcQuery::new(vpc.clone()), + ) + .await + { + Ok(Some((authz_vpc, vpc))) => { + crate::probes::vni__search__range__found!(|| { + (&id, u32::from(vpc.vni.0)) + }); + return Ok((authz_vpc, vpc)); + } + Err(e) => return Err(e), + Ok(None) => { + crate::probes::vni__search__range__empty!(|| (&id)); + debug!( + opctx.log, + "No VNIs available within current search range, retrying"; + "attempt" => i, + "vpc_name" => %vpc.identity.name, + "start_vni" => ?vni, + ); + } + } + } + + // We've failed to find a VNI after searching the entire range, so we'll + // return a 503 at this point. + error!( + opctx.log, + "failed to find a VNI after searching entire range"; + ); + Err(Error::unavail("Failed to find a free VNI for this VPC")) } + // Internal implementation for creating a VPC. + // + // This returns an optional VPC. If it is None, then we failed to insert a + // VPC specifically because there are no available VNIs. All other errors + // are returned in the `Result::Err` variant. async fn project_create_vpc_raw( &self, opctx: &OpContext, authz_project: &authz::Project, vpc_query: InsertVpcQuery, - ) -> Result<(authz::Vpc, Vpc), Error> { + ) -> Result, Error> { use db::schema::vpc::dsl; assert_eq!(authz_project.id(), vpc_query.vpc.project_id); @@ -312,30 +363,48 @@ impl DataStore { let project_id = vpc_query.vpc.project_id; let conn = self.pool_connection_authorized(opctx).await?; - let vpc: Vpc = Project::insert_resource( + let result: Result = Project::insert_resource( project_id, diesel::insert_into(dsl::vpc).values(vpc_query), ) .insert_and_get_result_async(&conn) - .await - .map_err(|e| match e { - AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { - type_name: ResourceType::Project, - lookup_type: LookupType::ById(project_id), - }, - AsyncInsertError::DatabaseError(e) => public_error_from_diesel( - e, - ErrorHandler::Conflict(ResourceType::Vpc, name.as_str()), - ), - })?; - Ok(( - authz::Vpc::new( - authz_project.clone(), - vpc.id(), - LookupType::ByName(vpc.name().to_string()), - ), - vpc, - )) + .await; + match result { + Ok(vpc) => Ok(Some(( + authz::Vpc::new( + authz_project.clone(), + vpc.id(), + LookupType::ByName(vpc.name().to_string()), + ), + vpc, + ))), + Err(AsyncInsertError::CollectionNotFound) => { + Err(Error::ObjectNotFound { + type_name: ResourceType::Project, + lookup_type: LookupType::ById(project_id), + }) + } + Err(AsyncInsertError::DatabaseError( + DieselError::DatabaseError( + DatabaseErrorKind::NotNullViolation, + info, + ), + )) if info + .message() + .starts_with("null value in column \"vni\"") => + { + // We failed the non-null check on the VNI column, which means + // we could not find a valid VNI in our search range. Return + // None instead to signal the error. + Ok(None) + } + Err(AsyncInsertError::DatabaseError(e)) => { + Err(public_error_from_diesel( + e, + ErrorHandler::Conflict(ResourceType::Vpc, name.as_str()), + )) + } + } } pub async fn project_update_vpc( @@ -1092,3 +1161,234 @@ impl DataStore { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::datastore::datastore_test; + use crate::db::model::Project; + use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; + use nexus_test_utils::db::test_setup_database; + use nexus_types::external_api::params; + use omicron_common::api::external; + use omicron_test_utils::dev; + use slog::info; + + // Test that we detect the right error condition and return None when we + // fail to insert a VPC due to VNI exhaustion. + // + // This is a bit awkward, but we'll test this by inserting a bunch of VPCs, + // and checking that we get the expected error response back from the + // `project_create_vpc_raw` call. + #[tokio::test] + async fn test_project_create_vpc_raw_returns_none_on_vni_exhaustion() { + usdt::register_probes().unwrap(); + let logctx = dev::test_setup_log( + "test_project_create_vpc_raw_returns_none_on_vni_exhaustion", + ); + let log = &logctx.log; + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // Create a project. + let project_params = params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: String::from("test project"), + }, + }; + let project = Project::new(Uuid::new_v4(), project_params); + let (authz_project, _) = datastore + .project_create(&opctx, project) + .await + .expect("failed to create project"); + + let starting_vni = 2048; + let description = String::from("test vpc"); + for vni in 0..=MAX_VNI_SEARCH_RANGE_SIZE { + // Create an incomplete VPC and make sure it has the next available + // VNI. + let name: external::Name = format!("vpc{vni}").parse().unwrap(); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = + Vni(external::Vni::try_from(starting_vni + vni).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating initial VPC"; + "index" => vni, + "vni" => ?this_vni, + ); + let query = InsertVpcQuery::new(incomplete_vpc); + let (_, db_vpc) = datastore + .project_create_vpc_raw(&opctx, &authz_project, query) + .await + .expect("failed to create initial set of VPCs") + .expect("expected an actual VPC"); + info!( + log, + "created VPC"; + "vpc" => ?db_vpc, + ); + } + + // At this point, we've filled all the VNIs starting from 2048. Let's + // try to allocate one more, also starting from that position. This + // should fail, because we've explicitly filled the entire range we'll + // search above. + let name: external::Name = "dead-vpc".parse().unwrap(); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = Vni(external::Vni::try_from(starting_vni).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating VPC when all VNIs are allocated"; + "vni" => ?this_vni, + ); + let query = InsertVpcQuery::new(incomplete_vpc); + let Ok(None) = datastore + .project_create_vpc_raw(&opctx, &authz_project, query) + .await + else { + panic!("Expected Ok(None) when creating a VPC without any available VNIs"); + }; + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + + // Test that we appropriately retry when there are no available VNIs. + // + // This is a bit awkward, but we'll test this by inserting a bunch of VPCs, + // and then check that we correctly retry + #[tokio::test] + async fn test_project_create_vpc_retries() { + usdt::register_probes().unwrap(); + let logctx = dev::test_setup_log("test_project_create_vpc_retries"); + let log = &logctx.log; + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // Create a project. + let project_params = params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: String::from("test project"), + }, + }; + let project = Project::new(Uuid::new_v4(), project_params); + let (authz_project, _) = datastore + .project_create(&opctx, project) + .await + .expect("failed to create project"); + + let starting_vni = 2048; + let description = String::from("test vpc"); + for vni in 0..=MAX_VNI_SEARCH_RANGE_SIZE { + // Create an incomplete VPC and make sure it has the next available + // VNI. + let name: external::Name = format!("vpc{vni}").parse().unwrap(); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = + Vni(external::Vni::try_from(starting_vni + vni).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating initial VPC"; + "index" => vni, + "vni" => ?this_vni, + ); + let query = InsertVpcQuery::new(incomplete_vpc); + let (_, db_vpc) = datastore + .project_create_vpc_raw(&opctx, &authz_project, query) + .await + .expect("failed to create initial set of VPCs") + .expect("expected an actual VPC"); + info!( + log, + "created VPC"; + "vpc" => ?db_vpc, + ); + } + + // Similar to the above test, we've fill all available VPCs starting at + // `starting_vni`. Let's attempt to allocate one beginning there, which + // _should_ fail and be internally retried. Note that we're using + // `project_create_vpc()` here instead of the raw version, to check that + // retry logic. + let name: external::Name = "dead-at-first-vpc".parse().unwrap(); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = Vni(external::Vni::try_from(starting_vni).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating VPC when all VNIs are allocated"; + "vni" => ?this_vni, + ); + match datastore + .project_create_vpc(&opctx, &authz_project, incomplete_vpc.clone()) + .await + { + Ok((_, vpc)) => { + assert_eq!(vpc.id(), incomplete_vpc.identity.id); + let expected_vni = starting_vni + MAX_VNI_SEARCH_RANGE_SIZE + 1; + assert_eq!(u32::from(vpc.vni.0), expected_vni); + info!(log, "successfully created VPC after retries"; "vpc" => ?vpc); + } + Err(e) => panic!("Unexpected error when inserting VPC: {e}"), + }; + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/queries/mod.rs b/nexus/db-queries/src/db/queries/mod.rs index cd48be61e3..a1022f9187 100644 --- a/nexus/db-queries/src/db/queries/mod.rs +++ b/nexus/db-queries/src/db/queries/mod.rs @@ -14,6 +14,7 @@ mod next_item; pub mod network_interface; pub mod region_allocation; pub mod virtual_provisioning_collection_update; +pub mod volume; pub mod vpc; pub mod vpc_subnet; diff --git a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs index b7271f3f49..0a383eb6f1 100644 --- a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs +++ b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs @@ -368,10 +368,12 @@ impl VirtualProvisioningCollectionUpdate { pub fn new_delete_instance( id: uuid::Uuid, + max_instance_gen: i64, cpus_diff: i64, ram_diff: ByteCount, project_id: uuid::Uuid, ) -> Self { + use crate::db::schema::instance::dsl as instance_dsl; use virtual_provisioning_collection::dsl as collection_dsl; use virtual_provisioning_resource::dsl as resource_dsl; @@ -379,9 +381,36 @@ impl VirtualProvisioningCollectionUpdate { // We should delete the record if it exists. DoUpdate::new_for_delete(id), // The query to actually delete the record. + // + // The filter condition here ensures that the provisioning record is + // only deleted if the corresponding instance has a generation + // number less than the supplied `max_instance_gen`. This allows a + // caller that is about to apply an instance update that will stop + // the instance and that bears generation G to avoid deleting + // resources if the instance generation was already advanced to or + // past G. + // + // If the relevant instance ID is not in the database, then some + // other operation must have ensured the instance was previously + // stopped (because that's the only way it could have been deleted), + // and that operation should have cleaned up the resources already, + // in which case there's nothing to do here. + // + // There is an additional "direct" filter on the target resource ID + // to avoid a full scan of the resource table. UnreferenceableSubquery( diesel::delete(resource_dsl::virtual_provisioning_resource) .filter(resource_dsl::id.eq(id)) + .filter( + resource_dsl::id.nullable().eq(instance_dsl::instance + .filter(instance_dsl::id.eq(id)) + .filter( + instance_dsl::state_generation + .lt(max_instance_gen), + ) + .select(instance_dsl::id) + .single_value()), + ) .returning(virtual_provisioning_resource::all_columns), ), // Within this project, silo, fleet... diff --git a/nexus/db-queries/src/db/queries/volume.rs b/nexus/db-queries/src/db/queries/volume.rs new file mode 100644 index 0000000000..31882dca89 --- /dev/null +++ b/nexus/db-queries/src/db/queries/volume.rs @@ -0,0 +1,497 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Helper queries for working with volumes. + +use crate::db; +use crate::db::pool::DbConnection; +use diesel::expression::is_aggregate; +use diesel::expression::ValidGrouping; +use diesel::pg::Pg; +use diesel::query_builder::AstPass; +use diesel::query_builder::Query; +use diesel::query_builder::QueryFragment; +use diesel::query_builder::QueryId; +use diesel::sql_types; +use diesel::Column; +use diesel::Expression; +use diesel::QueryResult; +use diesel::RunQueryDsl; +use uuid::Uuid; + +/// Produces a query fragment that will act as a filter for the data modifying +/// sub-queries of the "decrease crucible resource count and soft delete volume" CTE. +/// +/// The output should look like: +/// +/// ```sql +/// (SELECT CASE +/// WHEN volume.resources_to_clean_up is null then true +/// ELSE false +/// END +/// FROM volume WHERE id = '{}') +/// ``` +#[must_use = "Queries must be executed"] +struct ResourcesToCleanUpColumnIsNull { + volume_id: Uuid, +} + +impl ResourcesToCleanUpColumnIsNull { + pub fn new(volume_id: Uuid) -> Self { + Self { volume_id } + } +} + +impl QueryId for ResourcesToCleanUpColumnIsNull { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for ResourcesToCleanUpColumnIsNull { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::volume; + + out.push_sql("SELECT CASE WHEN "); + out.push_identifier(volume::dsl::resources_to_clean_up::NAME)?; + out.push_sql(" is null then true ELSE false END FROM "); + volume::dsl::volume.walk_ast(out.reborrow())?; + out.push_sql(" WHERE "); + out.push_identifier(volume::dsl::id::NAME)?; + out.push_sql(" = "); + out.push_bind_param::(&self.volume_id)?; + + Ok(()) + } +} + +impl Expression for ResourcesToCleanUpColumnIsNull { + type SqlType = sql_types::Bool; +} + +impl ValidGrouping + for ResourcesToCleanUpColumnIsNull +{ + type IsAggregate = is_aggregate::Never; +} + +/// Produces a query fragment that will conditionally reduce the volume +/// references for region_snapshot rows whose snapshot_addr column is part of a +/// list. +/// +/// The output should look like: +/// +/// ```sql +/// update region_snapshot set +/// volume_references = volume_references - 1, +/// deleting = case when volume_references = 1 +/// then true +/// else false +/// end +/// where +/// snapshot_addr in ('a1', 'a2', 'a3') and +/// volume_references >= 1 and +/// deleting = false and +/// () +/// returning * +/// ``` +#[must_use = "Queries must be executed"] +struct ConditionallyDecreaseReferences { + resources_to_clean_up_column_is_null_clause: ResourcesToCleanUpColumnIsNull, + snapshot_addrs: Vec, +} + +impl ConditionallyDecreaseReferences { + pub fn new(volume_id: Uuid, snapshot_addrs: Vec) -> Self { + Self { + resources_to_clean_up_column_is_null_clause: + ResourcesToCleanUpColumnIsNull::new(volume_id), + snapshot_addrs, + } + } +} + +impl QueryId for ConditionallyDecreaseReferences { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for ConditionallyDecreaseReferences { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::region_snapshot::dsl; + + out.push_sql("UPDATE "); + dsl::region_snapshot.walk_ast(out.reborrow())?; + out.push_sql(" SET "); + out.push_identifier(dsl::volume_references::NAME)?; + out.push_sql(" = "); + out.push_identifier(dsl::volume_references::NAME)?; + out.push_sql(" - 1, "); + out.push_identifier(dsl::deleting::NAME)?; + out.push_sql(" = CASE WHEN "); + out.push_identifier(dsl::volume_references::NAME)?; + out.push_sql(" = 1 THEN TRUE ELSE FALSE END WHERE "); + out.push_identifier(dsl::snapshot_addr::NAME)?; + out.push_sql(" IN ("); + + // If self.snapshot_addrs is empty, this query fragment will intentionally not update any + // region_snapshot rows. The rest of the CTE should still run to completion. + for (i, snapshot_addr) in self.snapshot_addrs.iter().enumerate() { + out.push_bind_param::(snapshot_addr)?; + if i == self.snapshot_addrs.len() - 1 { + out.push_sql(" "); + } else { + out.push_sql(", "); + } + } + + out.push_sql(") AND "); + out.push_identifier(dsl::volume_references::NAME)?; + out.push_sql(" >= 1 AND "); + out.push_identifier(dsl::deleting::NAME)?; + out.push_sql(" = false AND ( "); + self.resources_to_clean_up_column_is_null_clause + .walk_ast(out.reborrow())?; + out.push_sql(") RETURNING *"); + + Ok(()) + } +} + +impl Expression for ConditionallyDecreaseReferences { + type SqlType = sql_types::Array; +} + +impl ValidGrouping + for ConditionallyDecreaseReferences +{ + type IsAggregate = is_aggregate::Never; +} + +/// Produces a query fragment that will find all resources that can be cleaned +/// up as a result of a volume delete, and build a serialized JSON struct that +/// can be deserialized into a CrucibleResources::V3 variant. The output of this +/// will be written into the 'resources_to_clean_up` column of the volume being +/// soft-deleted. +/// +/// The output should look like: +/// +/// ```sql +/// json_build_object('V3', +/// json_build_object( +/// 'regions', (select json_agg(id) from region join t2 on region.id = t2.region_id where (t2.volume_references = 0 or t2.volume_references is null) and region.volume_id = ''), +/// 'region_snapshots', (select json_agg(json_build_object('dataset', dataset_id, 'region', region_id, 'snapshot', snapshot_id)) from t2 where t2.volume_references = 0) +/// ) +/// ) +/// ``` +/// +/// Note if json_agg is executing over zero rows, then the output is `null`, not +/// `[]`. For example, if the sub-query meant to return regions to clean up +/// returned zero rows, the output of json_build_object would look like: +/// +/// ```json +/// { +/// "V3": { +/// "regions": null, +/// ... +/// } +/// } +/// ``` +/// +/// Correctly handling `null` here is done in the deserializer for +/// CrucibleResourcesV3. +/// +/// A populated object should look like: +/// +/// ```json +/// { +/// "V3": { +/// "regions": [ +/// "9caae5bb-a212-4496-882a-af1ee242c62f", +/// "713c84ee-6b13-4301-b7a2-36debc7ee37e" +/// ], +/// "region_snapshots": [ +/// { +/// "dataset": "33ec5f07-5e7f-481e-966a-0fbc50d9ed3b", +/// "region": "1e2b1a75-9a58-4e5c-89a0-0cfd19ba055a", +/// "snapshot": "f7c8ed87-a67e-4d2b-8f35-3e8034de1c6f" +/// }, +/// { +/// "dataset": "5a16b1d6-7381-4c51-b49c-997624d43ead", +/// "region": "52b4c9bc-d1c9-4a3b-87c3-8e4501a883b0", +/// "snapshot": "2dd912e4-74db-409a-8d55-9795496cb320" +/// } +/// ] +/// } +/// } +/// ``` +#[must_use = "Queries must be executed"] +struct BuildJsonResourcesToCleanUp { + table: &'static str, + volume_id: Uuid, +} + +impl BuildJsonResourcesToCleanUp { + pub fn new(table: &'static str, volume_id: Uuid) -> Self { + Self { table, volume_id } + } +} + +impl QueryId for BuildJsonResourcesToCleanUp { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for BuildJsonResourcesToCleanUp { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::region::dsl as region_dsl; + use db::schema::region_snapshot::dsl as region_snapshot_dsl; + use db::schema::volume::dsl; + + out.push_sql("json_build_object('V3', "); + out.push_sql("json_build_object('regions', "); + out.push_sql("(SELECT json_agg("); + out.push_identifier(dsl::id::NAME)?; + out.push_sql(") FROM "); + region_dsl::region.walk_ast(out.reborrow())?; + out.push_sql(" JOIN "); + out.push_sql(self.table); + out.push_sql(" ON "); + out.push_identifier(region_dsl::id::NAME)?; + out.push_sql(" = "); + out.push_sql(self.table); + out.push_sql("."); + out.push_identifier(region_snapshot_dsl::region_id::NAME)?; // table's schema is equivalent to region_snapshot + out.push_sql(" WHERE ( "); + + out.push_sql(self.table); + out.push_sql("."); + out.push_identifier(region_snapshot_dsl::volume_references::NAME)?; + out.push_sql(" = 0 OR "); + out.push_sql(self.table); + out.push_sql("."); + out.push_identifier(region_snapshot_dsl::volume_references::NAME)?; + out.push_sql(" IS NULL"); + + out.push_sql(") AND "); + out.push_identifier(region_dsl::volume_id::NAME)?; + out.push_sql(" = "); + out.push_bind_param::(&self.volume_id)?; + + out.push_sql("), 'region_snapshots', ("); + out.push_sql("SELECT json_agg(json_build_object("); + out.push_sql("'dataset', "); + out.push_identifier(region_snapshot_dsl::dataset_id::NAME)?; + out.push_sql(", 'region', "); + out.push_identifier(region_snapshot_dsl::region_id::NAME)?; + out.push_sql(", 'snapshot', "); + out.push_identifier(region_snapshot_dsl::snapshot_id::NAME)?; + out.push_sql(")) from "); + out.push_sql(self.table); + out.push_sql(" where "); + out.push_sql(self.table); + out.push_sql("."); + out.push_identifier(region_snapshot_dsl::volume_references::NAME)?; + out.push_sql(" = 0)))"); + + Ok(()) + } +} + +impl ValidGrouping + for BuildJsonResourcesToCleanUp +{ + type IsAggregate = is_aggregate::Never; +} + +/// Produces a query fragment that will set the `resources_to_clean_up` column +/// of the volume being deleted if it is not set already. +/// +/// The output should look like: +/// +/// ```sql +/// update volume set +/// time_deleted = now(), +/// resources_to_clean_up = ( select ) +/// where id = '' and +/// () +/// returning volume.* +/// ``` +#[must_use = "Queries must be executed"] +struct ConditionallyUpdateVolume { + resources_to_clean_up_column_is_null_clause: ResourcesToCleanUpColumnIsNull, + build_json_resources_to_clean_up_query: BuildJsonResourcesToCleanUp, + volume_id: Uuid, +} + +impl ConditionallyUpdateVolume { + pub fn new(volume_id: Uuid, table: &'static str) -> Self { + Self { + resources_to_clean_up_column_is_null_clause: + ResourcesToCleanUpColumnIsNull::new(volume_id), + build_json_resources_to_clean_up_query: + BuildJsonResourcesToCleanUp::new(table, volume_id), + volume_id, + } + } +} + +impl QueryId for ConditionallyUpdateVolume { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for ConditionallyUpdateVolume { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::volume::dsl; + + out.push_sql("UPDATE "); + dsl::volume.walk_ast(out.reborrow())?; + out.push_sql(" SET "); + out.push_identifier(dsl::time_deleted::NAME)?; + out.push_sql(" = now(), "); + out.push_identifier(dsl::resources_to_clean_up::NAME)?; + out.push_sql(" = (SELECT "); + + self.build_json_resources_to_clean_up_query.walk_ast(out.reborrow())?; + + out.push_sql(") WHERE "); + out.push_identifier(dsl::id::NAME)?; + out.push_sql(" = "); + out.push_bind_param::(&self.volume_id)?; + out.push_sql(" AND ("); + + self.resources_to_clean_up_column_is_null_clause + .walk_ast(out.reborrow())?; + + out.push_sql(") RETURNING volume.*"); + + Ok(()) + } +} + +impl Expression for ConditionallyUpdateVolume { + type SqlType = diesel::sql_types::Array; +} + +impl ValidGrouping for ConditionallyUpdateVolume { + type IsAggregate = is_aggregate::Never; +} + +/// Produces a query fragment that will +/// +/// 1. decrease the number of references for each region snapshot that +/// a volume references +/// 2. soft-delete the volume +/// 3. record the resources to clean up as a serialized CrucibleResources +/// struct in volume's `resources_to_clean_up` column. +/// +/// The output should look like: +/// +/// ```sql +/// with UPDATED_REGION_SNAPSHOTS_TABLE as ( +/// UPDATE region_snapshot +/// ), +/// REGION_SNAPSHOTS_TO_CLEAN_UP_TABLE as ( +/// select * from UPDATED_REGION_SNAPSHOTS_TABLE where deleting = true and volume_references = 0 +/// ), +/// UPDATED_VOLUME_TABLE as ( +/// UPDATE volume +/// ) +/// select case +/// when volume.resources_to_clean_up is not null then volume.resources_to_clean_up +/// else (select resources_to_clean_up from UPDATED_VOLUME_TABLE where id = '') +/// end +/// from volume where id = ''; +/// ``` +#[must_use = "Queries must be executed"] +pub struct DecreaseCrucibleResourceCountAndSoftDeleteVolume { + conditionally_decrease_references: ConditionallyDecreaseReferences, + conditionally_update_volume_query: ConditionallyUpdateVolume, + volume_id: Uuid, +} + +impl DecreaseCrucibleResourceCountAndSoftDeleteVolume { + const UPDATED_REGION_SNAPSHOTS_TABLE: &str = "updated_region_snapshots"; + const REGION_SNAPSHOTS_TO_CLEAN_UP_TABLE: &str = + "region_snapshots_to_clean_up"; + const UPDATED_VOLUME_TABLE: &str = "updated_volume"; + + pub fn new(volume_id: Uuid, snapshot_addrs: Vec) -> Self { + Self { + conditionally_decrease_references: + ConditionallyDecreaseReferences::new(volume_id, snapshot_addrs), + conditionally_update_volume_query: ConditionallyUpdateVolume::new( + volume_id, + Self::REGION_SNAPSHOTS_TO_CLEAN_UP_TABLE, + ), + volume_id, + } + } +} + +impl QueryId for DecreaseCrucibleResourceCountAndSoftDeleteVolume { + type QueryId = (); + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl QueryFragment for DecreaseCrucibleResourceCountAndSoftDeleteVolume { + fn walk_ast<'a>(&'a self, mut out: AstPass<'_, 'a, Pg>) -> QueryResult<()> { + use db::schema::region_snapshot::dsl as rs_dsl; + use db::schema::volume::dsl; + + out.push_sql("WITH "); + out.push_sql(Self::UPDATED_REGION_SNAPSHOTS_TABLE); + out.push_sql(" as ("); + self.conditionally_decrease_references.walk_ast(out.reborrow())?; + out.push_sql("), "); + + out.push_sql(Self::REGION_SNAPSHOTS_TO_CLEAN_UP_TABLE); + out.push_sql(" AS (SELECT * FROM "); + out.push_sql(Self::UPDATED_REGION_SNAPSHOTS_TABLE); + out.push_sql(" WHERE "); + out.push_identifier(rs_dsl::deleting::NAME)?; + out.push_sql(" = TRUE AND "); + out.push_identifier(rs_dsl::volume_references::NAME)?; + out.push_sql(" = 0), "); + + out.push_sql(Self::UPDATED_VOLUME_TABLE); + out.push_sql(" AS ("); + self.conditionally_update_volume_query.walk_ast(out.reborrow())?; + out.push_sql(") "); + + out.push_sql("SELECT "); + dsl::volume.walk_ast(out.reborrow())?; + out.push_sql(".* FROM "); + dsl::volume.walk_ast(out.reborrow())?; + out.push_sql(" WHERE "); + out.push_identifier(dsl::id::NAME)?; + out.push_sql(" = "); + out.push_bind_param::(&self.volume_id)?; + + Ok(()) + } +} + +impl Expression for DecreaseCrucibleResourceCountAndSoftDeleteVolume { + type SqlType = diesel::sql_types::Array; +} + +impl ValidGrouping + for DecreaseCrucibleResourceCountAndSoftDeleteVolume +{ + type IsAggregate = is_aggregate::Never; +} + +impl RunQueryDsl + for DecreaseCrucibleResourceCountAndSoftDeleteVolume +{ +} + +type SelectableSql = < + >::SelectExpression as diesel::Expression +>::SqlType; + +impl Query for DecreaseCrucibleResourceCountAndSoftDeleteVolume { + type SqlType = SelectableSql; +} diff --git a/nexus/db-queries/src/db/queries/vpc.rs b/nexus/db-queries/src/db/queries/vpc.rs index b1ac8fe1e1..c29a51adb0 100644 --- a/nexus/db-queries/src/db/queries/vpc.rs +++ b/nexus/db-queries/src/db/queries/vpc.rs @@ -245,15 +245,7 @@ struct NextVni { impl NextVni { fn new(vni: Vni) -> Self { - let base_u32 = u32::from(vni.0); - // The valid range is [0, 1 << 24], so the maximum shift is whatever - // gets us to 1 << 24, and the minimum is whatever gets us back to the - // minimum guest VNI. - let max_shift = i64::from(external::Vni::MAX_VNI - base_u32); - let min_shift = i64::from( - -i32::try_from(base_u32 - external::Vni::MIN_GUEST_VNI) - .expect("Expected a valid VNI at this point"), - ); + let VniShifts { min_shift, max_shift } = VniShifts::new(vni); let generator = DefaultShiftGenerator { base: vni, max_shift, min_shift }; let inner = NextItem::new_unscoped(generator); @@ -278,3 +270,208 @@ impl NextVni { } delegate_query_fragment_impl!(NextVni); + +// Helper type to compute the shift for a `NextItem` query to find VNIs. +#[derive(Clone, Copy, Debug, PartialEq)] +struct VniShifts { + // The minimum `ShiftGenerator` shift. + min_shift: i64, + // The maximum `ShiftGenerator` shift. + max_shift: i64, +} + +/// Restrict the search for a VNI to a small range. +/// +/// VNIs are pretty sparsely allocated (the number of VPCs), and the range is +/// quite large (24 bits). To avoid memory issues, we'll restrict a search +/// for an available VNI to a small range starting from the random starting +/// VNI. +// +// NOTE: This is very small for tests, to ensure we can accurately test the +// failure mode where there are no available VNIs. +#[cfg(not(test))] +pub const MAX_VNI_SEARCH_RANGE_SIZE: u32 = 2048; +#[cfg(test)] +pub const MAX_VNI_SEARCH_RANGE_SIZE: u32 = 10; + +// Ensure that we cannot search a range that extends beyond the valid guest VNI +// range. +static_assertions::const_assert!( + MAX_VNI_SEARCH_RANGE_SIZE + <= (external::Vni::MAX_VNI - external::Vni::MIN_GUEST_VNI) +); + +impl VniShifts { + fn new(vni: Vni) -> Self { + let base_u32 = u32::from(vni.0); + let range_end = base_u32 + MAX_VNI_SEARCH_RANGE_SIZE; + + // Clamp the maximum shift at the distance to the maximum allowed VNI, + // or the maximum of the range. + let max_shift = i64::from( + (external::Vni::MAX_VNI - base_u32).min(MAX_VNI_SEARCH_RANGE_SIZE), + ); + + // And any remaining part of the range wraps around starting at the + // beginning. + let min_shift = -i64::from( + range_end.checked_sub(external::Vni::MAX_VNI).unwrap_or(0), + ); + Self { min_shift, max_shift } + } +} + +/// An iterator yielding sequential starting VNIs. +/// +/// The VPC insertion query requires a search for the next available VNI, using +/// the `NextItem` query. We limit the search for each query to avoid memory +/// issues on any one query. If we fail to find a VNI, we need to search the +/// next range. This iterator yields the starting positions for the `NextItem` +/// query, so that the entire range can be search in chunks until a free VNI is +/// found. +// +// NOTE: It's technically possible for this to lead to searching the very +// initial portion of the range twice. If we end up wrapping around so that the +// last position yielded by this iterator is `start - x`, then we'll end up +// searching from `start - x` to `start + (MAX_VNI_SEARCH_RANGE_SIZE - x)`, and +// so search those first few after `start` again. This is both innocuous and +// really unlikely. +#[derive(Clone, Copy, Debug)] +pub struct VniSearchIter { + start: u32, + current: u32, + has_wrapped: bool, +} + +impl VniSearchIter { + pub const STEP_SIZE: u32 = MAX_VNI_SEARCH_RANGE_SIZE; + + /// Create a search range, starting from the provided VNI. + pub fn new(start: external::Vni) -> Self { + let start = u32::from(start); + Self { start, current: start, has_wrapped: false } + } +} + +impl std::iter::Iterator for VniSearchIter { + type Item = external::Vni; + + fn next(&mut self) -> Option { + // If we've wrapped around and the computed position is beyond where we + // started, then the ite + if self.has_wrapped && self.current > self.start { + return None; + } + + // Compute the next position. + // + // Make sure we wrap around to the mininum guest VNI. Note that we + // consider the end of the range inclusively, so we subtract one in the + // offset below to end up _at_ the min guest VNI. + let mut next = self.current + MAX_VNI_SEARCH_RANGE_SIZE; + if next > external::Vni::MAX_VNI { + next -= external::Vni::MAX_VNI; + next += external::Vni::MIN_GUEST_VNI - 1; + self.has_wrapped = true; + } + let current = self.current; + self.current = next; + Some(external::Vni::try_from(current).unwrap()) + } +} + +#[cfg(test)] +mod tests { + use super::external; + use super::Vni; + use super::VniSearchIter; + use super::VniShifts; + use super::MAX_VNI_SEARCH_RANGE_SIZE; + + // Ensure that when the search range lies entirely within the range of VNIs, + // we search from the start VNI through the maximum allowed range size. + #[test] + fn test_vni_shift_no_wrapping() { + let vni = Vni(external::Vni::try_from(2048).unwrap()); + let VniShifts { min_shift, max_shift } = VniShifts::new(vni); + assert_eq!(min_shift, 0); + assert_eq!(max_shift, i64::from(MAX_VNI_SEARCH_RANGE_SIZE)); + assert_eq!(max_shift - min_shift, i64::from(MAX_VNI_SEARCH_RANGE_SIZE)); + } + + // Ensure that we wrap correctly, when the starting VNI happens to land + // quite close to the end of the allowed range. + #[test] + fn test_vni_shift_with_wrapping() { + let offset = 5; + let vni = + Vni(external::Vni::try_from(external::Vni::MAX_VNI - offset) + .unwrap()); + let VniShifts { min_shift, max_shift } = VniShifts::new(vni); + assert_eq!(min_shift, -i64::from(MAX_VNI_SEARCH_RANGE_SIZE - offset)); + assert_eq!(max_shift, i64::from(offset)); + assert_eq!(max_shift - min_shift, i64::from(MAX_VNI_SEARCH_RANGE_SIZE)); + } + + #[test] + fn test_vni_search_iter_steps() { + let start = external::Vni::try_from(2048).unwrap(); + let mut it = VniSearchIter::new(start); + let next = it.next().unwrap(); + assert_eq!(next, start); + let next = it.next().unwrap(); + assert_eq!( + next, + external::Vni::try_from( + u32::from(start) + MAX_VNI_SEARCH_RANGE_SIZE + ) + .unwrap() + ); + } + + #[test] + fn test_vni_search_iter_full_count() { + let start = + external::Vni::try_from(external::Vni::MIN_GUEST_VNI).unwrap(); + + let last = VniSearchIter::new(start).last().unwrap(); + println!("{:?}", last); + + pub const fn div_ceil(x: u32, y: u32) -> u32 { + let d = x / y; + let r = x % y; + if r > 0 && y > 0 { + d + 1 + } else { + d + } + } + const N_EXPECTED: u32 = div_ceil( + external::Vni::MAX_VNI - external::Vni::MIN_GUEST_VNI, + MAX_VNI_SEARCH_RANGE_SIZE, + ); + let count = u32::try_from(VniSearchIter::new(start).count()).unwrap(); + assert_eq!(count, N_EXPECTED); + } + + #[test] + fn test_vni_search_iter_wrapping() { + // Start from just before the end of the range. + let start = + external::Vni::try_from(external::Vni::MAX_VNI - 1).unwrap(); + let mut it = VniSearchIter::new(start); + + // We should yield that start position first. + let next = it.next().unwrap(); + assert_eq!(next, start); + + // The next value should be wrapped around to the beginning. + // + // Subtract 2 because we _include_ the max VNI in the search range. + let next = it.next().unwrap(); + assert_eq!( + u32::from(next), + external::Vni::MIN_GUEST_VNI + MAX_VNI_SEARCH_RANGE_SIZE - 2 + ); + } +} diff --git a/nexus/db-queries/src/lib.rs b/nexus/db-queries/src/lib.rs index 29c33039ff..a693f7ff42 100644 --- a/nexus/db-queries/src/lib.rs +++ b/nexus/db-queries/src/lib.rs @@ -17,3 +17,22 @@ extern crate newtype_derive; #[cfg(test)] #[macro_use] extern crate diesel; + +#[usdt::provider(provider = "nexus__db__queries")] +mod probes { + // Fires before we start a search over a range for a VNI. + // + // Includes the starting VNI and the size of the range being searched. + fn vni__search__range__start( + _: &usdt::UniqueId, + start_vni: u32, + size: u32, + ) { + } + + // Fires when we successfully find a VNI. + fn vni__search__range__found(_: &usdt::UniqueId, vni: u32) {} + + // Fires when we fail to find a VNI in the provided range. + fn vni__search__range__empty(_: &usdt::UniqueId) {} +} diff --git a/nexus/src/app/bgp.rs b/nexus/src/app/bgp.rs new file mode 100644 index 0000000000..e800d72bdd --- /dev/null +++ b/nexus/src/app/bgp.rs @@ -0,0 +1,162 @@ +use crate::app::authz; +use crate::external_api::params; +use nexus_db_model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig}; +use nexus_db_queries::context::OpContext; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::{ + BgpImportedRouteIpv4, BgpPeerStatus, CreateResult, DeleteResult, Ipv4Net, + ListResultVec, LookupResult, NameOrId, +}; + +impl super::Nexus { + pub async fn bgp_config_set( + &self, + opctx: &OpContext, + config: ¶ms::BgpConfigCreate, + ) -> CreateResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = self.db_datastore.bgp_config_set(opctx, config).await?; + Ok(result) + } + + pub async fn bgp_config_get( + &self, + opctx: &OpContext, + name_or_id: NameOrId, + ) -> LookupResult { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + self.db_datastore.bgp_config_get(opctx, &name_or_id).await + } + + pub async fn bgp_config_list( + &self, + opctx: &OpContext, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + self.db_datastore.bgp_config_list(opctx, pagparams).await + } + + pub async fn bgp_config_delete( + &self, + opctx: &OpContext, + sel: ¶ms::BgpConfigSelector, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = self.db_datastore.bgp_config_delete(opctx, sel).await?; + Ok(result) + } + + pub async fn bgp_create_announce_set( + &self, + opctx: &OpContext, + announce: ¶ms::BgpAnnounceSetCreate, + ) -> CreateResult<(BgpAnnounceSet, Vec)> { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = + self.db_datastore.bgp_create_announce_set(opctx, announce).await?; + Ok(result) + } + + pub async fn bgp_announce_list( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + self.db_datastore.bgp_announce_list(opctx, sel).await + } + + pub async fn bgp_delete_announce_set( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = + self.db_datastore.bgp_delete_announce_set(opctx, sel).await?; + Ok(result) + } + + pub async fn bgp_peer_status( + &self, + opctx: &OpContext, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + let mut result = Vec::new(); + for (switch, client) in &self.mg_clients { + let router_info = match client.inner.get_routers().await { + Ok(result) => result.into_inner(), + Err(e) => { + error!( + self.log, + "failed to get routers from {switch}: {e}" + ); + continue; + } + }; + + for r in &router_info { + for (addr, info) in &r.peers { + let Ok(addr) = addr.parse() else { + continue; + }; + result.push(BgpPeerStatus { + switch: *switch, + addr, + local_asn: r.asn, + remote_asn: info.asn.unwrap_or(0), + state: info.state.into(), + state_duration_millis: info.duration_millis, + }); + } + } + } + Ok(result) + } + + pub async fn bgp_imported_routes_ipv4( + &self, + opctx: &OpContext, + sel: ¶ms::BgpRouteSelector, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + let mut result = Vec::new(); + for (switch, client) in &self.mg_clients { + let imported: Vec = match client + .inner + .get_imported4(&mg_admin_client::types::GetImported4Request { + asn: sel.asn, + }) + .await + { + Ok(result) => result + .into_inner() + .into_iter() + .map(|x| BgpImportedRouteIpv4 { + switch: *switch, + prefix: Ipv4Net( + ipnetwork::Ipv4Network::new( + x.prefix.value, + x.prefix.length, + ) + .unwrap(), + ), + nexthop: x.nexthop, + id: x.id, + }) + .collect(), + Err(e) => { + error!( + self.log, + "failed to get BGP imported from {switch}: {e}" + ); + continue; + } + }; + + result.extend_from_slice(&imported); + } + Ok(result) + } +} diff --git a/nexus/src/app/image.rs b/nexus/src/app/image.rs index ac51773b05..8fa9308c1d 100644 --- a/nexus/src/app/image.rs +++ b/nexus/src/app/image.rs @@ -4,8 +4,8 @@ //! Images (both project and silo scoped) -use super::Unimpl; use crate::external_api::params; +use nexus_db_queries::authn; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -27,6 +27,8 @@ use std::str::FromStr; use std::sync::Arc; use uuid::Uuid; +use super::sagas; + impl super::Nexus { pub(crate) async fn image_lookup<'a>( &'a self, @@ -356,26 +358,33 @@ impl super::Nexus { } } - // TODO-MVP: Implement pub(crate) async fn image_delete( self: &Arc, opctx: &OpContext, image_lookup: &ImageLookup<'_>, ) -> DeleteResult { - match image_lookup { + let image_param: sagas::image_delete::ImageParam = match image_lookup { ImageLookup::ProjectImage(lookup) => { - lookup.lookup_for(authz::Action::Delete).await?; + let (_, _, authz_image, image) = + lookup.fetch_for(authz::Action::Delete).await?; + sagas::image_delete::ImageParam::Project { authz_image, image } } ImageLookup::SiloImage(lookup) => { - lookup.lookup_for(authz::Action::Delete).await?; + let (_, authz_image, image) = + lookup.fetch_for(authz::Action::Delete).await?; + sagas::image_delete::ImageParam::Silo { authz_image, image } } }; - let error = Error::InternalError { - internal_message: "Endpoint not implemented".to_string(), + + let saga_params = sagas::image_delete::Params { + serialized_authn: authn::saga::Serialized::for_opctx(opctx), + image_param, }; - Err(self - .unimplemented_todo(opctx, Unimpl::ProtectedLookup(error)) - .await) + + self.execute_saga::(saga_params) + .await?; + + Ok(()) } /// Converts a project scoped image into a silo scoped image diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 592e1f0492..17d033c5a0 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -987,6 +987,12 @@ impl super::Nexus { source_nat, external_ips, firewall_rules, + dhcp_config: sled_agent_client::types::DhcpConfig { + dns_servers: self.external_dns_servers.clone(), + // TODO: finish designing instance DNS + host_domain: None, + search_domains: Vec::new(), + }, disks: disk_reqs, cloud_init_bytes: Some(base64::Engine::encode( &base64::engine::general_purpose::STANDARD, @@ -1284,6 +1290,14 @@ impl super::Nexus { "propolis_id" => %propolis_id, "vmm_state" => ?new_runtime_state.vmm_state); + // Grab the current state of the instance in the DB to reason about + // whether this update is stale or not. + let (.., authz_instance, db_instance) = + LookupPath::new(&opctx, &self.db_datastore) + .instance_id(*instance_id) + .fetch() + .await?; + // Update OPTE and Dendrite if the instance's active sled assignment // changed or a migration was retired. If these actions fail, sled agent // is expected to retry this update. @@ -1297,12 +1311,6 @@ impl super::Nexus { // // In the future, this should be replaced by a call to trigger a // networking state update RPW. - let (.., authz_instance, db_instance) = - LookupPath::new(&opctx, &self.db_datastore) - .instance_id(*instance_id) - .fetch() - .await?; - self.ensure_updated_instance_network_config( opctx, &authz_instance, @@ -1311,6 +1319,27 @@ impl super::Nexus { ) .await?; + // If the supplied instance state indicates that the instance no longer + // has an active VMM, attempt to delete the virtual provisioning record + // + // As with updating networking state, this must be done before + // committing the new runtime state to the database: once the DB is + // written, a new start saga can arrive and start the instance, which + // will try to create its own virtual provisioning charges, which will + // race with this operation. + if new_runtime_state.instance_state.propolis_id.is_none() { + self.db_datastore + .virtual_provisioning_collection_delete_instance( + opctx, + *instance_id, + db_instance.project_id, + i64::from(db_instance.ncpus.0 .0), + db_instance.memory, + (&new_runtime_state.instance_state.gen).into(), + ) + .await?; + } + // Write the new instance and VMM states back to CRDB. This needs to be // done before trying to clean up the VMM, since the datastore will only // allow a VMM to be marked as deleted if it is already in a terminal @@ -1331,7 +1360,20 @@ impl super::Nexus { // If the VMM is now in a terminal state, make sure its resources get // cleaned up. - if let Ok((_, true)) = result { + // + // For idempotency, only check to see if the update was successfully + // processed and ignore whether the VMM record was actually updated. + // This is required to handle the case where this routine is called + // once, writes the terminal VMM state, fails before all per-VMM + // resources are released, returns a retriable error, and is retried: + // the per-VMM resources still need to be cleaned up, but the DB update + // will return Ok(_, false) because the database was already updated. + // + // Unlike the pre-update cases, it is legal to do this cleanup *after* + // committing state to the database, because a terminated VMM cannot be + // reused (restarting or migrating its former instance will use new VMM + // IDs). + if result.is_ok() { let propolis_terminated = matches!( new_runtime_state.vmm_state.state, InstanceState::Destroyed | InstanceState::Failed diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 8d61832d2a..ef8132451a 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -20,13 +20,14 @@ use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use omicron_common::address::DENDRITE_PORT; +use omicron_common::address::MGD_PORT; use omicron_common::address::MGS_PORT; use omicron_common::api::external::Error; use omicron_common::api::internal::shared::SwitchLocation; use omicron_common::nexus_config::RegionAllocationStrategy; use slog::Logger; use std::collections::HashMap; -use std::net::Ipv6Addr; +use std::net::{IpAddr, Ipv6Addr}; use std::sync::Arc; use uuid::Uuid; @@ -34,6 +35,7 @@ use uuid::Uuid; // by resource. mod address_lot; pub(crate) mod background; +mod bgp; mod certificate; mod device_auth; mod disk; @@ -114,6 +116,10 @@ pub struct Nexus { /// External dropshot servers external_server: std::sync::Mutex>, + /// External dropshot server that listens on the internal network to allow + /// connections from the tech port; see RFD 431. + techport_external_server: std::sync::Mutex>, + /// Internal dropshot server internal_server: std::sync::Mutex>, @@ -149,9 +155,18 @@ pub struct Nexus { /// DNS resolver Nexus uses to resolve an external host external_resolver: Arc, + /// DNS servers used in `external_resolver`, used to provide DNS servers to + /// instances via DHCP + // TODO: This needs to be moved to the database. + // https://github.com/oxidecomputer/omicron/issues/3732 + external_dns_servers: Vec, + /// Mapping of SwitchLocations to their respective Dendrite Clients dpd_clients: HashMap>, + /// Map switch location to maghemite admin clients. + mg_clients: HashMap>, + /// Background tasks background_tasks: background::BackgroundTasks, @@ -206,7 +221,13 @@ impl Nexus { let mut dpd_clients: HashMap> = HashMap::new(); - // Currently static dpd configuration mappings are still required for testing + let mut mg_clients: HashMap< + SwitchLocation, + Arc, + > = HashMap::new(); + + // Currently static dpd configuration mappings are still required for + // testing for (location, config) in &config.pkg.dendrite { let address = config.address.ip().to_string(); let port = config.address.port(); @@ -216,6 +237,11 @@ impl Nexus { ); dpd_clients.insert(*location, Arc::new(dpd_client)); } + for (location, config) in &config.pkg.mgd { + let mg_client = mg_admin_client::Client::new(&log, config.address) + .map_err(|e| format!("mg admin client: {e}"))?; + mg_clients.insert(*location, Arc::new(mg_client)); + } if config.pkg.dendrite.is_empty() { loop { let result = resolver @@ -249,6 +275,42 @@ impl Nexus { } } } + if config.pkg.mgd.is_empty() { + loop { + let result = resolver + // TODO this should be ServiceName::Mgd, but in the upgrade + // path, that does not exist because RSS has not + // created it. So we just piggyback on Dendrite's SRV + // record. + .lookup_all_ipv6(ServiceName::Dendrite) + .await + .map_err(|e| format!("Cannot lookup mgd addresses: {e}")); + match result { + Ok(addrs) => { + let mappings = map_switch_zone_addrs( + &log.new(o!("component" => "Nexus")), + addrs, + ) + .await; + for (location, addr) in &mappings { + let port = MGD_PORT; + let mgd_client = mg_admin_client::Client::new( + &log, + std::net::SocketAddr::new((*addr).into(), port), + ) + .map_err(|e| format!("mg admin client: {e}"))?; + mg_clients.insert(*location, Arc::new(mgd_client)); + } + break; + } + Err(e) => { + warn!(log, "Failed to lookup mgd address: {e}"); + tokio::time::sleep(std::time::Duration::from_secs(1)) + .await; + } + } + } + } // Connect to clickhouse - but do so lazily. // Clickhouse may not be executing when Nexus starts. @@ -309,6 +371,7 @@ impl Nexus { sec_client: Arc::clone(&sec_client), recovery_task: std::sync::Mutex::new(None), external_server: std::sync::Mutex::new(None), + techport_external_server: std::sync::Mutex::new(None), internal_server: std::sync::Mutex::new(None), populate_status, timeseries_client, @@ -329,7 +392,12 @@ impl Nexus { samael_max_issue_delay: std::sync::Mutex::new(None), internal_resolver: resolver, external_resolver, + external_dns_servers: config + .deployment + .external_dns_servers + .clone(), dpd_clients, + mg_clients, background_tasks, default_region_allocation_strategy: config .pkg @@ -339,6 +407,12 @@ impl Nexus { // TODO-cleanup all the extra Arcs here seems wrong let nexus = Arc::new(nexus); + let bootstore_opctx = OpContext::for_background( + log.new(o!("component" => "Bootstore")), + Arc::clone(&authz), + authn::Context::internal_api(), + Arc::clone(&db_datastore), + ); let opctx = OpContext::for_background( log.new(o!("component" => "SagaRecoverer")), Arc::clone(&authz), @@ -378,6 +452,12 @@ impl Nexus { for task in task_nexus.background_tasks.driver.tasks() { task_nexus.background_tasks.driver.activate(task); } + if let Err(e) = task_nexus + .initial_bootstore_sync(&bootstore_opctx) + .await + { + error!(task_log, "failed to run bootstore sync: {e}"); + } } Err(_) => { error!(task_log, "populate failed"); @@ -441,6 +521,7 @@ impl Nexus { pub(crate) async fn set_servers( &self, external_server: DropshotServer, + techport_external_server: DropshotServer, internal_server: DropshotServer, ) { // If any servers already exist, close them. @@ -448,6 +529,10 @@ impl Nexus { // Insert the new servers. self.external_server.lock().unwrap().replace(external_server); + self.techport_external_server + .lock() + .unwrap() + .replace(techport_external_server); self.internal_server.lock().unwrap().replace(internal_server); } @@ -456,6 +541,11 @@ impl Nexus { if let Some(server) = external_server { server.close().await?; } + let techport_external_server = + self.techport_external_server.lock().unwrap().take(); + if let Some(server) = techport_external_server { + server.close().await?; + } let internal_server = self.internal_server.lock().unwrap().take(); if let Some(server) = internal_server { server.close().await?; diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 67da485c46..bed690f839 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -5,11 +5,14 @@ //! Rack management use super::silo::silo_dns_name; +use crate::external_api::params; use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; +use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -33,17 +36,21 @@ use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::IdentityMetadataCreateParams; -use omicron_common::api::external::IpNet; -use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::Name; use omicron_common::api::external::NameOrId; use omicron_common::api::internal::shared::ExternalPortDiscovery; +use sled_agent_client::types::EarlyNetworkConfigBody; +use sled_agent_client::types::{ + BgpConfig, BgpPeerConfig, EarlyNetworkConfig, PortConfigV1, + RackNetworkConfigV1, RouteConfig as SledRouteConfig, +}; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; +use std::net::Ipv4Addr; use std::str::FromStr; use uuid::Uuid; @@ -188,10 +195,18 @@ impl super::Nexus { mapped_fleet_roles, }; + let rack_network_config = request.rack_network_config.as_ref().ok_or( + Error::InvalidRequest { + message: "cannot initialize a rack without a network config" + .into(), + }, + )?; + self.db_datastore .rack_set_initialized( opctx, RackInit { + rack_subnet: rack_network_config.rack_subnet.into(), rack_id, services: request.services, datasets, @@ -264,7 +279,9 @@ impl super::Nexus { let qsfp_ports: Vec = all_ports .iter() - .filter(|port| port.starts_with("qsfp")) + .filter(|port| { + matches!(port, dpd_client::types::PortId::Qsfp(_)) + }) .map(|port| port.to_string().parse().unwrap()) .collect(); @@ -381,7 +398,7 @@ impl super::Nexus { })?; for (idx, uplink_config) in - rack_network_config.uplinks.iter().enumerate() + rack_network_config.ports.iter().enumerate() { let switch = uplink_config.switch.to_string(); let switch_location = Name::from_str(&switch).map_err(|e| { @@ -450,35 +467,40 @@ impl super::Nexus { addresses: HashMap::new(), }; - let uplink_address = - IpNet::V4(Ipv4Net(uplink_config.uplink_cidr)); - let address = Address { - address_lot: NameOrId::Name(address_lot_name.clone()), - address: uplink_address, - }; - port_settings_params.addresses.insert( - "phy0".to_string(), - AddressConfig { addresses: vec![address] }, - ); - - let dst = IpNet::from_str("0.0.0.0/0").map_err(|e| { - Error::internal_error(&format!( - "failed to parse provided default route CIDR: {e}" - )) - })?; - - let gw = IpAddr::V4(uplink_config.gateway_ip); - let vid = uplink_config.uplink_vid; - let route = Route { dst, gw, vid }; - - port_settings_params.routes.insert( - "phy0".to_string(), - RouteConfig { routes: vec![route] }, - ); + let addresses: Vec
= uplink_config + .addresses + .iter() + .map(|a| Address { + address_lot: NameOrId::Name(address_lot_name.clone()), + address: (*a).into(), + }) + .collect(); + + port_settings_params + .addresses + .insert("phy0".to_string(), AddressConfig { addresses }); + + let routes: Vec = uplink_config + .routes + .iter() + .map(|r| Route { + dst: r.destination.into(), + gw: r.nexthop, + vid: None, + }) + .collect(); + + port_settings_params + .routes + .insert("phy0".to_string(), RouteConfig { routes }); match self .db_datastore - .switch_port_settings_create(opctx, &port_settings_params) + .switch_port_settings_create( + opctx, + &port_settings_params, + None, + ) .await { Ok(_) | Err(Error::ObjectAlreadyExists { .. }) => Ok(()), @@ -499,9 +521,7 @@ impl super::Nexus { opctx, rack_id, switch_location.into(), - Name::from_str(&uplink_config.uplink_port) - .unwrap() - .into(), + Name::from_str(&uplink_config.port).unwrap().into(), ) .await?; @@ -516,6 +536,7 @@ impl super::Nexus { } // TODO - https://github.com/oxidecomputer/omicron/issues/3277 // record port speed }; + self.initial_bootstore_sync(&opctx).await?; Ok(()) } @@ -549,4 +570,153 @@ impl super::Nexus { tokio::time::sleep(std::time::Duration::from_secs(2)).await; } } + + pub(crate) async fn initial_bootstore_sync( + &self, + opctx: &OpContext, + ) -> Result<(), Error> { + let mut rack = self.rack_lookup(opctx, &self.rack_id).await?; + if rack.rack_subnet.is_some() { + return Ok(()); + } + let addr = self + .sled_list(opctx, &DataPageParams::max_page()) + .await? + .get(0) + .ok_or(Error::InternalError { + internal_message: "no sleds at time of bootstore sync".into(), + })? + .address(); + + let sa = sled_agent_client::Client::new( + &format!("http://{}", addr), + self.log.clone(), + ); + + let result = sa + .read_network_bootstore_config_cache() + .await + .map_err(|e| Error::InternalError { + internal_message: format!("read bootstore network config: {e}"), + })? + .into_inner(); + + rack.rack_subnet = + result.body.rack_network_config.map(|x| x.rack_subnet.into()); + + self.datastore().update_rack_subnet(opctx, &rack).await?; + + Ok(()) + } + + pub(crate) async fn bootstore_network_config( + &self, + opctx: &OpContext, + ) -> Result { + let rack = self.rack_lookup(opctx, &self.rack_id).await?; + + let subnet = match rack.rack_subnet { + Some(IpNetwork::V6(subnet)) => subnet, + Some(IpNetwork::V4(_)) => { + return Err(Error::InternalError { + internal_message: "rack subnet not IPv6".into(), + }) + } + None => { + return Err(Error::InternalError { + internal_message: "rack subnet not set".into(), + }) + } + }; + + let db_ports = self.active_port_settings(opctx).await?; + let mut ports = Vec::new(); + let mut bgp = Vec::new(); + for (port, info) in &db_ports { + let mut peer_info = Vec::new(); + for p in &info.bgp_peers { + let bgp_config = + self.bgp_config_get(&opctx, p.bgp_config_id.into()).await?; + let announcements = self + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: bgp_config.bgp_announce_set_id.into(), + }, + ) + .await?; + let addr = match p.addr { + ipnetwork::IpNetwork::V4(addr) => addr, + ipnetwork::IpNetwork::V6(_) => continue, //TODO v6 + }; + peer_info.push((p, bgp_config.asn.0, addr.ip())); + bgp.push(BgpConfig { + asn: bgp_config.asn.0, + originate: announcements + .iter() + .filter_map(|a| match a.network { + IpNetwork::V4(net) => Some(net.into()), + //TODO v6 + _ => None, + }) + .collect(), + }); + } + + let p = PortConfigV1 { + routes: info + .routes + .iter() + .map(|r| SledRouteConfig { + destination: r.dst, + nexthop: r.gw.ip(), + }) + .collect(), + addresses: info.addresses.iter().map(|a| a.address).collect(), + bgp_peers: peer_info + .iter() + .map(|(_p, asn, addr)| BgpPeerConfig { + addr: *addr, + asn: *asn, + port: port.port_name.clone(), + }) + .collect(), + switch: port.switch_location.parse().unwrap(), + port: port.port_name.clone(), + uplink_port_fec: info + .links + .get(0) //TODO breakout support + .map(|l| l.fec) + .unwrap_or(SwitchLinkFec::None) + .into(), + uplink_port_speed: info + .links + .get(0) //TODO breakout support + .map(|l| l.speed) + .unwrap_or(SwitchLinkSpeed::Speed100G) + .into(), + }; + + ports.push(p); + } + + let result = EarlyNetworkConfig { + generation: 0, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: Vec::new(), //TODO + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: subnet, + //TODO(ry) you are here. We need to remove these too. They are + // inconsistent with a generic set of addresses on ports. + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports, + bgp, + }), + }, + }; + + Ok(result) + } } diff --git a/nexus/src/app/sagas/image_delete.rs b/nexus/src/app/sagas/image_delete.rs new file mode 100644 index 0000000000..1d88ff17af --- /dev/null +++ b/nexus/src/app/sagas/image_delete.rs @@ -0,0 +1,124 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use super::{ActionRegistry, NexusActionContext, NexusSaga}; +use crate::app::sagas; +use crate::app::sagas::declare_saga_actions; +use nexus_db_queries::{authn, authz, db}; +use serde::Deserialize; +use serde::Serialize; +use steno::ActionError; +use steno::Node; +use uuid::Uuid; + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) enum ImageParam { + Project { authz_image: authz::ProjectImage, image: db::model::ProjectImage }, + + Silo { authz_image: authz::SiloImage, image: db::model::SiloImage }, +} + +impl ImageParam { + fn volume_id(&self) -> Uuid { + match self { + ImageParam::Project { image, .. } => image.volume_id, + + ImageParam::Silo { image, .. } => image.volume_id, + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) struct Params { + pub serialized_authn: authn::saga::Serialized, + pub image_param: ImageParam, +} + +declare_saga_actions! { + image_delete; + DELETE_IMAGE_RECORD -> "no_result1" { + + sid_delete_image_record + } +} + +#[derive(Debug)] +pub(crate) struct SagaImageDelete; +impl NexusSaga for SagaImageDelete { + const NAME: &'static str = "image-delete"; + type Params = Params; + + fn register_actions(registry: &mut ActionRegistry) { + image_delete_register_actions(registry); + } + + fn make_saga_dag( + params: &Self::Params, + mut builder: steno::DagBuilder, + ) -> Result { + builder.append(delete_image_record_action()); + + const DELETE_VOLUME_PARAMS: &'static str = "delete_volume_params"; + + let volume_delete_params = sagas::volume_delete::Params { + serialized_authn: params.serialized_authn.clone(), + volume_id: params.image_param.volume_id(), + }; + builder.append(Node::constant( + DELETE_VOLUME_PARAMS, + serde_json::to_value(&volume_delete_params).map_err(|e| { + super::SagaInitError::SerializeError( + String::from("volume_id"), + e, + ) + })?, + )); + + let make_volume_delete_dag = || { + let subsaga_builder = steno::DagBuilder::new(steno::SagaName::new( + sagas::volume_delete::SagaVolumeDelete::NAME, + )); + sagas::volume_delete::create_dag(subsaga_builder) + }; + builder.append(steno::Node::subsaga( + "delete_volume", + make_volume_delete_dag()?, + DELETE_VOLUME_PARAMS, + )); + + Ok(builder.build()?) + } +} + +// image delete saga: action implementations + +async fn sid_delete_image_record( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + match params.image_param { + ImageParam::Project { authz_image, image } => { + osagactx + .datastore() + .project_image_delete(&opctx, &authz_image, image) + .await + .map_err(ActionError::action_failed)?; + } + + ImageParam::Silo { authz_image, image } => { + osagactx + .datastore() + .silo_image_delete(&opctx, &authz_image, image) + .await + .map_err(ActionError::action_failed)?; + } + } + + Ok(()) +} diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index 5d55aaf0fe..153e0323e7 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -13,7 +13,6 @@ use crate::external_api::params; use nexus_db_model::NetworkInterfaceKind; use nexus_db_queries::db::identity::Resource; use nexus_db_queries::db::lookup::LookupPath; -use nexus_db_queries::db::model::ByteCount as DbByteCount; use nexus_db_queries::db::queries::network_interface::InsertError as InsertNicError; use nexus_db_queries::{authn, authz, db}; use nexus_defaults::DEFAULT_PRIMARY_NIC_NAME; @@ -75,10 +74,6 @@ struct DiskAttachParams { declare_saga_actions! { instance_create; - VIRTUAL_RESOURCES_ACCOUNT -> "no_result" { - + sic_account_virtual_resources - - sic_account_virtual_resources_undo - } CREATE_INSTANCE_RECORD -> "instance_record" { + sic_create_instance_record - sic_delete_instance_record @@ -131,7 +126,6 @@ impl NexusSaga for SagaInstanceCreate { })?, )); - builder.append(virtual_resources_account_action()); builder.append(create_instance_record_action()); // Helper function for appending subsagas to our parent saga. @@ -728,56 +722,6 @@ async fn ensure_instance_disk_attach_state( Ok(()) } -async fn sic_account_virtual_resources( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let instance_id = sagactx.lookup::("instance_id")?; - - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - osagactx - .datastore() - .virtual_provisioning_collection_insert_instance( - &opctx, - instance_id, - params.project_id, - i64::from(params.create_params.ncpus.0), - DbByteCount(params.create_params.memory), - ) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - -async fn sic_account_virtual_resources_undo( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let instance_id = sagactx.lookup::("instance_id")?; - - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - osagactx - .datastore() - .virtual_provisioning_collection_delete_instance( - &opctx, - instance_id, - params.project_id, - i64::from(params.create_params.ncpus.0), - DbByteCount(params.create_params.memory), - ) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - async fn sic_create_instance_record( sagactx: NexusActionContext, ) -> Result { diff --git a/nexus/src/app/sagas/instance_delete.rs b/nexus/src/app/sagas/instance_delete.rs index 7da497136e..1605465c74 100644 --- a/nexus/src/app/sagas/instance_delete.rs +++ b/nexus/src/app/sagas/instance_delete.rs @@ -9,7 +9,6 @@ use super::NexusActionContext; use super::NexusSaga; use crate::app::sagas::declare_saga_actions; use nexus_db_queries::{authn, authz, db}; -use nexus_types::identity::Resource; use omicron_common::api::external::{Error, ResourceType}; use omicron_common::api::internal::shared::SwitchLocation; use serde::Deserialize; @@ -40,9 +39,6 @@ declare_saga_actions! { DEALLOCATE_EXTERNAL_IP -> "no_result3" { + sid_deallocate_external_ip } - VIRTUAL_RESOURCES_ACCOUNT -> "no_result4" { - + sid_account_virtual_resources - } } // instance delete saga: definition @@ -64,7 +60,6 @@ impl NexusSaga for SagaInstanceDelete { builder.append(instance_delete_record_action()); builder.append(delete_network_interfaces_action()); builder.append(deallocate_external_ip_action()); - builder.append(virtual_resources_account_action()); Ok(builder.build()?) } } @@ -135,30 +130,6 @@ async fn sid_deallocate_external_ip( Ok(()) } -async fn sid_account_virtual_resources( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - - osagactx - .datastore() - .virtual_provisioning_collection_delete_instance( - &opctx, - params.instance.id(), - params.instance.project_id, - i64::from(params.instance.ncpus.0 .0), - params.instance.memory, - ) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - #[cfg(test)] mod test { use crate::{ diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index 068d2e7005..76773d6369 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -52,6 +52,11 @@ declare_saga_actions! { - sis_move_to_starting_undo } + ADD_VIRTUAL_RESOURCES -> "virtual_resources" { + + sis_account_virtual_resources + - sis_account_virtual_resources_undo + } + // TODO(#3879) This can be replaced with an action that triggers the NAT RPW // once such an RPW is available. DPD_ENSURE -> "dpd_ensure" { @@ -98,6 +103,7 @@ impl NexusSaga for SagaInstanceStart { builder.append(alloc_propolis_ip_action()); builder.append(create_vmm_record_action()); builder.append(mark_as_starting_action()); + builder.append(add_virtual_resources_action()); builder.append(dpd_ensure_action()); builder.append(v2p_ensure_action()); builder.append(ensure_registered_action()); @@ -305,6 +311,66 @@ async fn sis_move_to_starting_undo( Ok(()) } +async fn sis_account_virtual_resources( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let instance_id = params.db_instance.id(); + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + osagactx + .datastore() + .virtual_provisioning_collection_insert_instance( + &opctx, + instance_id, + params.db_instance.project_id, + i64::from(params.db_instance.ncpus.0 .0), + nexus_db_model::ByteCount(*params.db_instance.memory), + ) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + +async fn sis_account_virtual_resources_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let instance_id = params.db_instance.id(); + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let started_record = + sagactx.lookup::("started_record")?; + + osagactx + .datastore() + .virtual_provisioning_collection_delete_instance( + &opctx, + instance_id, + params.db_instance.project_id, + i64::from(params.db_instance.ncpus.0 .0), + nexus_db_model::ByteCount(*params.db_instance.memory), + // Use the next instance generation number as the generation limit + // to ensure the provisioning counters are released. (The "mark as + // starting" undo step will "publish" this new state generation when + // it moves the instance back to Stopped.) + (&started_record.runtime().gen.next()).into(), + ) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + async fn sis_dpd_ensure( sagactx: NexusActionContext, ) -> Result<(), ActionError> { diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs index 88778e3573..83e0e9b8b4 100644 --- a/nexus/src/app/sagas/mod.rs +++ b/nexus/src/app/sagas/mod.rs @@ -22,6 +22,7 @@ use uuid::Uuid; pub mod disk_create; pub mod disk_delete; pub mod finalize_disk; +pub mod image_delete; pub mod import_blocks_from_url; mod instance_common; pub mod instance_create; @@ -35,7 +36,6 @@ pub mod snapshot_create; pub mod snapshot_delete; pub mod switch_port_settings_apply; pub mod switch_port_settings_clear; -pub mod switch_port_settings_update; pub mod test_saga; pub mod volume_delete; pub mod volume_remove_rop; @@ -167,6 +167,9 @@ fn make_action_registry() -> ActionRegistry { &mut registry, ); ::register_actions(&mut registry); + ::register_actions( + &mut registry, + ); #[cfg(test)] ::register_actions(&mut registry); diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index 687613f0cc..fb06dc5fc0 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -7,6 +7,7 @@ use crate::app::sagas::retry_until_known_result; use crate::app::sagas::{ declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError, }; +use crate::Nexus; use anyhow::Error; use db::datastore::SwitchPortSettingsCombinedResult; use dpd_client::types::{ @@ -15,18 +16,37 @@ use dpd_client::types::{ }; use dpd_client::{Ipv4Cidr, Ipv6Cidr}; use ipnetwork::IpNetwork; +use mg_admin_client::types::Prefix4; +use mg_admin_client::types::{ApplyRequest, BgpPeerConfig, BgpRoute}; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed, NETWORK_KEY}; +use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::UpdatePrecondition; use nexus_db_queries::{authn, db}; -use omicron_common::api::external::{self, NameOrId}; -use omicron_common::api::internal::shared::SwitchLocation; +use nexus_types::external_api::params; +use omicron_common::api::external::{self, DataPageParams, NameOrId}; +use omicron_common::api::internal::shared::{ + ParseSwitchLocationError, SwitchLocation, +}; use serde::{Deserialize, Serialize}; +use sled_agent_client::types::PortConfigV1; +use sled_agent_client::types::RouteConfig; +use sled_agent_client::types::{BgpConfig, EarlyNetworkConfig}; +use sled_agent_client::types::{ + BgpPeerConfig as OmicronBgpPeerConfig, HostPortConfig, +}; use std::collections::HashMap; use std::net::IpAddr; +use std::net::SocketAddrV6; use std::str::FromStr; use std::sync::Arc; use steno::ActionError; use uuid::Uuid; +// This is more of an implementation detail of the BGP implementation. It +// defines the maximum time the peering engine will wait for external messages +// before breaking to check for shutdown conditions. +const BGP_SESSION_RESOLUTION: u64 = 100; + // switch port settings apply saga: input parameters #[derive(Debug, Deserialize, Serialize)] @@ -52,6 +72,18 @@ declare_saga_actions! { + spa_ensure_switch_port_settings - spa_undo_ensure_switch_port_settings } + ENSURE_SWITCH_PORT_UPLINK -> "ensure_switch_port_uplink" { + + spa_ensure_switch_port_uplink + - spa_undo_ensure_switch_port_uplink + } + ENSURE_SWITCH_PORT_BGP_SETTINGS -> "ensure_switch_port_bgp_settings" { + + spa_ensure_switch_port_bgp_settings + - spa_undo_ensure_switch_port_bgp_settings + } + ENSURE_SWITCH_PORT_BOOTSTORE_NETWORK_SETTINGS -> "ensure_switch_port_bootstore_network_settings" { + + spa_ensure_switch_port_bootstore_network_settings + - spa_undo_ensure_switch_port_bootstore_network_settings + } } // switch port settings apply saga: definition @@ -74,6 +106,9 @@ impl NexusSaga for SagaSwitchPortSettingsApply { builder.append(associate_switch_port_action()); builder.append(get_switch_port_settings_action()); builder.append(ensure_switch_port_settings_action()); + builder.append(ensure_switch_port_uplink_action()); + builder.append(ensure_switch_port_bgp_settings_action()); + builder.append(ensure_switch_port_bootstore_network_settings_action()); Ok(builder.build()?) } } @@ -91,10 +126,10 @@ async fn spa_associate_switch_port( ); // first get the current association so we fall back to this on failure - let port = nexus - .get_switch_port(&opctx, params.switch_port_id) - .await - .map_err(ActionError::action_failed)?; + let port = + nexus.get_switch_port(&opctx, params.switch_port_id).await.map_err( + |e| ActionError::action_failed(format!("get switch port: {e}")), + )?; // update the switch port settings association nexus @@ -105,7 +140,11 @@ async fn spa_associate_switch_port( UpdatePrecondition::DontCare, ) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!( + "set switch port settings id {e}" + )) + })?; Ok(port.port_settings_id) } @@ -127,7 +166,9 @@ async fn spa_get_switch_port_settings( &NameOrId::Id(params.switch_port_settings_id), ) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!("get switch port settings: {e}")) + })?; Ok(port_settings) } @@ -142,22 +183,42 @@ pub(crate) fn api_to_dpd_port_settings( v6_routes: HashMap::new(), }; - // TODO handle breakouts - // https://github.com/oxidecomputer/omicron/issues/3062 + //TODO breakouts let link_id = LinkId(0); - let link_settings = LinkSettings { - // TODO Allow user to configure link properties - // https://github.com/oxidecomputer/omicron/issues/3061 - params: LinkCreate { - autoneg: false, - kr: false, - fec: PortFec::None, - speed: PortSpeed::Speed100G, - }, - addrs: settings.addresses.iter().map(|a| a.address.ip()).collect(), - }; - dpd_port_settings.links.insert(link_id.to_string(), link_settings); + for l in settings.links.iter() { + dpd_port_settings.links.insert( + link_id.to_string(), + LinkSettings { + params: LinkCreate { + autoneg: false, + kr: false, + fec: match l.fec { + SwitchLinkFec::Firecode => PortFec::Firecode, + SwitchLinkFec::Rs => PortFec::Rs, + SwitchLinkFec::None => PortFec::None, + }, + speed: match l.speed { + SwitchLinkSpeed::Speed0G => PortSpeed::Speed0G, + SwitchLinkSpeed::Speed1G => PortSpeed::Speed1G, + SwitchLinkSpeed::Speed10G => PortSpeed::Speed10G, + SwitchLinkSpeed::Speed25G => PortSpeed::Speed25G, + SwitchLinkSpeed::Speed40G => PortSpeed::Speed40G, + SwitchLinkSpeed::Speed50G => PortSpeed::Speed50G, + SwitchLinkSpeed::Speed100G => PortSpeed::Speed100G, + SwitchLinkSpeed::Speed200G => PortSpeed::Speed200G, + SwitchLinkSpeed::Speed400G => PortSpeed::Speed400G, + }, + }, + //TODO won't work for breakouts + addrs: settings + .addresses + .iter() + .map(|a| a.address.ip()) + .collect(), + }, + ); + } for r in &settings.routes { match &r.dst { @@ -173,11 +234,7 @@ pub(crate) fn api_to_dpd_port_settings( dpd_port_settings.v4_routes.insert( Ipv4Cidr { prefix: n.ip(), prefix_len: n.prefix() } .to_string(), - RouteSettingsV4 { - link_id: link_id.0, - nexthop: gw, - vid: r.vid.map(Into::into), - }, + vec![RouteSettingsV4 { link_id: link_id.0, nexthop: gw }], ); } IpNetwork::V6(n) => { @@ -192,11 +249,7 @@ pub(crate) fn api_to_dpd_port_settings( dpd_port_settings.v6_routes.insert( Ipv6Cidr { prefix: n.ip(), prefix_len: n.prefix() } .to_string(), - RouteSettingsV6 { - link_id: link_id.0, - nexthop: gw, - vid: r.vid.map(Into::into), - }, + vec![RouteSettingsV6 { link_id: link_id.0, nexthop: gw }], ); } } @@ -214,20 +267,40 @@ async fn spa_ensure_switch_port_settings( let settings = sagactx .lookup::("switch_port_settings")?; - let port_id: PortId = PortId::from_str(¶ms.switch_port_name) - .map_err(|e| ActionError::action_failed(e.to_string()))?; + let port_id: PortId = + PortId::from_str(¶ms.switch_port_name).map_err(|e| { + ActionError::action_failed(format!("parse port id: {e}")) + })?; let dpd_client: Arc = select_dendrite_client(&sagactx).await?; - let dpd_port_settings = api_to_dpd_port_settings(&settings) - .map_err(ActionError::action_failed)?; + let dpd_port_settings = + api_to_dpd_port_settings(&settings).map_err(|e| { + ActionError::action_failed(format!( + "translate api port settings to dpd port settings: {e}", + )) + })?; retry_until_known_result(log, || async { dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await }) .await - .map_err(|e| ActionError::action_failed(e.to_string()))?; + .map_err(|e| match e { + progenitor_client::Error::ErrorResponse(ref er) => { + if er.status().is_client_error() { + ActionError::action_failed(format!( + "bad request: dpd port settings apply {}", + er.message, + )) + } else { + ActionError::action_failed(format!( + "dpd port settings apply {e}" + )) + } + } + _ => ActionError::action_failed(format!("dpd port settings apply {e}")), + })?; Ok(()) } @@ -270,10 +343,16 @@ async fn spa_undo_ensure_switch_port_settings( let settings = nexus .switch_port_settings_get(&opctx, &NameOrId::Id(id)) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!("switch port settings get: {e}")) + })?; - let dpd_port_settings = api_to_dpd_port_settings(&settings) - .map_err(ActionError::action_failed)?; + let dpd_port_settings = + api_to_dpd_port_settings(&settings).map_err(|e| { + ActionError::action_failed(format!( + "translate api to dpd port settings {e}" + )) + })?; retry_until_known_result(log, || async { dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await @@ -284,6 +363,326 @@ async fn spa_undo_ensure_switch_port_settings( Ok(()) } +async fn spa_ensure_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let settings = sagactx + .lookup::("switch_port_settings") + .map_err(|e| { + ActionError::action_failed(format!( + "lookup switch port settings: {e}" + )) + })?; + + ensure_switch_port_bgp_settings(sagactx, settings).await +} + +pub(crate) async fn ensure_switch_port_bgp_settings( + sagactx: NexusActionContext, + settings: SwitchPortSettingsCombinedResult, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let mg_client: Arc = + select_mg_client(&sagactx).await.map_err(|e| { + ActionError::action_failed(format!("select mg client: {e}")) + })?; + + let mut bgp_peer_configs = Vec::new(); + + for peer in settings.bgp_peers { + let config = nexus + .bgp_config_get(&opctx, peer.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("get bgp config: {e}")) + })?; + + let announcements = nexus + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: NameOrId::Id(config.bgp_announce_set_id), + }, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get bgp announcements: {e}" + )) + })?; + + // TODO picking the first configured address by default, but this needs + // to be something that can be specified in the API. + let nexthop = match settings.addresses.get(0) { + Some(switch_port_addr) => Ok(switch_port_addr.address.ip()), + None => Err(ActionError::action_failed( + "at least one address required for bgp peering".to_string(), + )), + }?; + + let nexthop = match nexthop { + IpAddr::V4(nexthop) => Ok(nexthop), + IpAddr::V6(_) => Err(ActionError::action_failed( + "IPv6 nexthop not yet supported".to_string(), + )), + }?; + + let mut prefixes = Vec::new(); + for a in &announcements { + let value = match a.network.ip() { + IpAddr::V4(value) => Ok(value), + IpAddr::V6(_) => Err(ActionError::action_failed( + "IPv6 announcement not yet supported".to_string(), + )), + }?; + prefixes.push(Prefix4 { value, length: a.network.prefix() }); + } + + let bpc = BgpPeerConfig { + asn: *config.asn, + name: format!("{}", peer.addr.ip()), //TODO user defined name? + host: format!("{}:179", peer.addr.ip()), + hold_time: peer.hold_time.0.into(), + idle_hold_time: peer.idle_hold_time.0.into(), + delay_open: peer.delay_open.0.into(), + connect_retry: peer.connect_retry.0.into(), + keepalive: peer.keepalive.0.into(), + resolution: BGP_SESSION_RESOLUTION, + routes: vec![BgpRoute { nexthop, prefixes }], + }; + + bgp_peer_configs.push(bpc); + } + + mg_client + .inner + .bgp_apply(&ApplyRequest { + peer_group: params.switch_port_name.clone(), + peers: bgp_peer_configs, + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("apply bgp settings: {e}")) + })?; + + Ok(()) +} +async fn spa_undo_ensure_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + use mg_admin_client::types::DeleteNeighborRequest; + + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let settings = sagactx + .lookup::("switch_port_settings") + .map_err(|e| { + ActionError::action_failed(format!( + "lookup switch port settings (bgp undo): {e}" + )) + })?; + + let mg_client: Arc = + select_mg_client(&sagactx).await.map_err(|e| { + ActionError::action_failed(format!("select mg client (undo): {e}")) + })?; + + for peer in settings.bgp_peers { + let config = nexus + .bgp_config_get(&opctx, peer.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete bgp config: {e}")) + })?; + + mg_client + .inner + .delete_neighbor(&DeleteNeighborRequest { + asn: *config.asn, + addr: peer.addr.ip(), + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete neighbor: {e}")) + })?; + } + + Ok(()) +} + +async fn spa_ensure_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + let mut config = + nexus.bootstore_network_config(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "read nexus bootstore network config: {e}" + )) + })?; + + let generation = nexus + .datastore() + .bump_bootstore_generation(&opctx, NETWORK_KEY.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "bump bootstore network generation number: {e}" + )) + })?; + + config.generation = generation as u64; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} + +async fn spa_undo_ensure_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + // The overall saga update failed but the bootstore udpate succeeded. + // Between now and then other updates may have happened which prevent us + // from simply undoing the changes we did before, as we may inadvertently + // roll back changes at the intersection of this failed update and other + // succesful updates. The only thing we can really do here is attempt a + // complete update of the bootstore network settings based on the current + // state in the Nexus databse which, we assume to be consistent at any point + // in time. + + let nexus = sagactx.user_data().nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + let config = nexus.bootstore_network_config(&opctx).await?; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} + +async fn spa_ensure_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + ensure_switch_port_uplink(sagactx, false, None).await +} + +async fn spa_undo_ensure_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), Error> { + Ok(ensure_switch_port_uplink(sagactx, true, None).await?) +} + +pub(crate) async fn ensure_switch_port_uplink( + sagactx: NexusActionContext, + skip_self: bool, + inject: Option, +) -> Result<(), ActionError> { + let params = sagactx.saga_params::()?; + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + + let switch_port = nexus + .get_switch_port(&opctx, params.switch_port_id) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for uplink: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err(|e| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + })?; + + let mut uplinks: Vec = Vec::new(); + + // The sled agent uplinks interface is an all or nothing interface, so we + // need to get all the uplink configs for all the ports. + let active_ports = + nexus.active_port_settings(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "get active switch port settings: {e}" + )) + })?; + + for (port, info) in &active_ports { + // Since we are undoing establishing uplinks for the settings + // associated with this port we skip adding this ports uplinks + // to the list - effectively removing them. + if skip_self && port.id == switch_port.id { + continue; + } + uplinks.push(HostPortConfig { + port: port.port_name.clone(), + addrs: info.addresses.iter().map(|a| a.address).collect(), + }) + } + + if let Some(id) = inject { + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let settings = nexus + .switch_port_settings_get(&opctx, &id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port settings for injection: {e}" + )) + })?; + uplinks.push(HostPortConfig { + port: params.switch_port_name.clone(), + addrs: settings.addresses.iter().map(|a| a.address).collect(), + }) + } + + let sc = switch_sled_agent(switch_location, &sagactx).await?; + sc.uplink_ensure(&sled_agent_client::types::SwitchPorts { uplinks }) + .await + .map_err(|e| { + ActionError::action_failed(format!("ensure uplink: {e}")) + })?; + + Ok(()) +} + // a common route representation for dendrite and port settings #[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)] pub(crate) struct Route { @@ -316,7 +715,11 @@ async fn spa_disassociate_switch_port( UpdatePrecondition::Value(params.switch_port_settings_id), ) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!( + "set switch port settings id for disassociate: {e}" + )) + })?; Ok(()) } @@ -335,12 +738,21 @@ pub(crate) async fn select_dendrite_client( let switch_port = nexus .get_switch_port(&opctx, params.switch_port_id) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for dendrite client selection {e}" + )) + })?; + let switch_location: SwitchLocation = - switch_port - .switch_location - .parse() - .map_err(ActionError::action_failed)?; + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + let dpd_client: Arc = osagactx .nexus() .dpd_clients @@ -353,3 +765,283 @@ pub(crate) async fn select_dendrite_client( .clone(); Ok(dpd_client) } + +pub(crate) async fn select_mg_client( + sagactx: &NexusActionContext, +) -> Result, ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let switch_port = nexus + .get_switch_port(&opctx, params.switch_port_id) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for mg client selection: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + + let mg_client: Arc = osagactx + .nexus() + .mg_clients + .get(&switch_location) + .ok_or_else(|| { + ActionError::action_failed(format!( + "requested switch not available: {switch_location}" + )) + })? + .clone(); + Ok(mg_client) +} + +pub(crate) async fn get_scrimlet_address( + _location: SwitchLocation, + nexus: &Arc, +) -> Result { + /* TODO this depends on DNS entries only coming from RSS, it's broken + on the upgrade path + nexus + .resolver() + .await + .lookup_socket_v6(ServiceName::Scrimlet(location)) + .await + .map_err(|e| e.to_string()) + .map_err(|e| { + ActionError::action_failed(format!( + "scrimlet dns lookup failed {e}", + )) + }) + */ + let opctx = &nexus.opctx_for_internal_api(); + Ok(nexus + .sled_list(opctx, &DataPageParams::max_page()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get_scrimlet_address: failed to list sleds: {e}" + )) + })? + .into_iter() + .find(|x| x.is_scrimlet()) + .ok_or(ActionError::action_failed( + "get_scrimlet_address: no scrimlets found".to_string(), + ))? + .address()) +} + +#[derive(Clone, Debug)] +pub struct EarlyNetworkPortUpdate { + port: PortConfigV1, + bgp_configs: Vec, +} + +pub(crate) async fn bootstore_update( + nexus: &Arc, + opctx: &OpContext, + switch_port_id: Uuid, + switch_port_name: &str, + settings: &SwitchPortSettingsCombinedResult, +) -> Result { + let switch_port = + nexus.get_switch_port(&opctx, switch_port_id).await.map_err(|e| { + ActionError::action_failed(format!( + "get switch port for uplink: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + + let mut peer_info = Vec::new(); + let mut bgp_configs = Vec::new(); + for p in &settings.bgp_peers { + let bgp_config = nexus + .bgp_config_get(&opctx, p.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("get bgp config: {e}")) + })?; + + let announcements = nexus + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: NameOrId::Id(bgp_config.bgp_announce_set_id), + }, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get bgp announcements: {e}" + )) + })?; + + peer_info.push((p, bgp_config.asn.0)); + bgp_configs.push(BgpConfig { + asn: bgp_config.asn.0, + originate: announcements + .iter() + .filter_map(|a| match a.network { + IpNetwork::V4(net) => Some(net.into()), + //TODO v6 + _ => None, + }) + .collect(), + }); + } + + let update = EarlyNetworkPortUpdate { + port: PortConfigV1 { + routes: settings + .routes + .iter() + .map(|r| RouteConfig { destination: r.dst, nexthop: r.gw.ip() }) + .collect(), + addresses: settings.addresses.iter().map(|a| a.address).collect(), + switch: switch_location, + port: switch_port_name.into(), + uplink_port_fec: settings + .links + .get(0) + .map(|l| l.fec) + .unwrap_or(SwitchLinkFec::None) + .into(), + uplink_port_speed: settings + .links + .get(0) + .map(|l| l.speed) + .unwrap_or(SwitchLinkSpeed::Speed100G) + .into(), + bgp_peers: peer_info + .iter() + .filter_map(|(p, asn)| { + //TODO v6 + match p.addr.ip() { + IpAddr::V4(addr) => Some(OmicronBgpPeerConfig { + asn: *asn, + port: switch_port_name.into(), + addr, + }), + IpAddr::V6(_) => { + warn!(opctx.log, "IPv6 peers not yet supported"); + None + } + } + }) + .collect(), + }, + bgp_configs, + }; + + Ok(update) +} + +pub(crate) async fn read_bootstore_config( + sa: &sled_agent_client::Client, +) -> Result { + Ok(sa + .read_network_bootstore_config_cache() + .await + .map_err(|e| { + ActionError::action_failed(format!( + "read bootstore network config: {e}" + )) + })? + .into_inner()) +} + +pub(crate) async fn write_bootstore_config( + sa: &sled_agent_client::Client, + config: &EarlyNetworkConfig, +) -> Result<(), ActionError> { + sa.write_network_bootstore_config(config).await.map_err(|e| { + ActionError::action_failed(format!( + "write bootstore network config: {e}" + )) + })?; + Ok(()) +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct BootstoreNetworkPortChange { + previous_port_config: Option, + changed_bgp_configs: Vec, + added_bgp_configs: Vec, +} + +pub(crate) fn apply_bootstore_update( + config: &mut EarlyNetworkConfig, + update: &EarlyNetworkPortUpdate, +) -> Result { + let mut change = BootstoreNetworkPortChange::default(); + + let rack_net_config = match &mut config.body.rack_network_config { + Some(cfg) => cfg, + None => { + return Err(ActionError::action_failed( + "rack network config not yet initialized".to_string(), + )) + } + }; + + for port in &mut rack_net_config.ports { + if port.port == update.port.port { + change.previous_port_config = Some(port.clone()); + *port = update.port.clone(); + break; + } + } + if change.previous_port_config.is_none() { + rack_net_config.ports.push(update.port.clone()); + } + + for updated_bgp in &update.bgp_configs { + let mut exists = false; + for resident_bgp in &mut rack_net_config.bgp { + if resident_bgp.asn == updated_bgp.asn { + change.changed_bgp_configs.push(resident_bgp.clone()); + *resident_bgp = updated_bgp.clone(); + exists = true; + break; + } + } + if !exists { + change.added_bgp_configs.push(updated_bgp.clone()); + } + } + rack_net_config.bgp.extend_from_slice(&change.added_bgp_configs); + + Ok(change) +} + +pub(crate) async fn switch_sled_agent( + location: SwitchLocation, + sagactx: &NexusActionContext, +) -> Result { + let nexus = sagactx.user_data().nexus(); + let sled_agent_addr = get_scrimlet_address(location, nexus).await?; + Ok(sled_agent_client::Client::new( + &format!("http://{}", sled_agent_addr), + sagactx.user_data().log().clone(), + )) +} diff --git a/nexus/src/app/sagas/switch_port_settings_clear.rs b/nexus/src/app/sagas/switch_port_settings_clear.rs index 0c0f4ec01b..14544b0f55 100644 --- a/nexus/src/app/sagas/switch_port_settings_clear.rs +++ b/nexus/src/app/sagas/switch_port_settings_clear.rs @@ -5,17 +5,25 @@ use super::switch_port_settings_apply::select_dendrite_client; use super::NexusActionContext; use crate::app::sagas::retry_until_known_result; -use crate::app::sagas::switch_port_settings_apply::api_to_dpd_port_settings; +use crate::app::sagas::switch_port_settings_apply::{ + api_to_dpd_port_settings, apply_bootstore_update, bootstore_update, + ensure_switch_port_bgp_settings, ensure_switch_port_uplink, + read_bootstore_config, select_mg_client, switch_sled_agent, + write_bootstore_config, +}; use crate::app::sagas::{ declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError, }; use anyhow::Error; use dpd_client::types::PortId; +use mg_admin_client::types::DeleteNeighborRequest; +use nexus_db_model::NETWORK_KEY; use nexus_db_queries::authn; use nexus_db_queries::db::datastore::UpdatePrecondition; -use omicron_common::api::external::{self, NameOrId}; +use omicron_common::api::external::{self, NameOrId, SwitchLocation}; use serde::{Deserialize, Serialize}; use std::str::FromStr; +use std::sync::Arc; use steno::ActionError; use uuid::Uuid; @@ -36,6 +44,18 @@ declare_saga_actions! { + spa_clear_switch_port_settings - spa_undo_clear_switch_port_settings } + CLEAR_SWITCH_PORT_UPLINK -> "clear_switch_port_uplink" { + + spa_clear_switch_port_uplink + - spa_undo_clear_switch_port_uplink + } + CLEAR_SWITCH_PORT_BGP_SETTINGS -> "clear_switch_port_bgp_settings" { + + spa_clear_switch_port_bgp_settings + - spa_undo_clear_switch_port_bgp_settings + } + CLEAR_SWITCH_PORT_BOOTSTORE_NETWORK_SETTINGS -> "clear_switch_port_bootstore_network_settings" { + + spa_clear_switch_port_bootstore_network_settings + - spa_undo_clear_switch_port_bootstore_network_settings + } } #[derive(Debug)] @@ -54,6 +74,9 @@ impl NexusSaga for SagaSwitchPortSettingsClear { ) -> Result { builder.append(disassociate_switch_port_action()); builder.append(clear_switch_port_settings_action()); + builder.append(clear_switch_port_uplink_action()); + builder.append(clear_switch_port_bgp_settings_action()); + builder.append(clear_switch_port_bootstore_network_settings_action()); Ok(builder.build()?) } } @@ -181,3 +204,185 @@ async fn spa_undo_clear_switch_port_settings( Ok(()) } + +async fn spa_clear_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + ensure_switch_port_uplink(sagactx, true, None).await +} + +async fn spa_undo_clear_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let id = sagactx + .lookup::>("original_switch_port_settings_id") + .map_err(|e| external::Error::internal_error(&e.to_string()))?; + + Ok(ensure_switch_port_uplink(sagactx, false, id).await?) +} + +async fn spa_clear_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = sagactx + .lookup::>("original_switch_port_settings_id") + .map_err(|e| { + ActionError::action_failed(format!( + "original port settings id lookup: {e}" + )) + })?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = nexus + .switch_port_settings_get(&opctx, &NameOrId::Id(id)) + .await + .map_err(ActionError::action_failed)?; + + let mg_client: Arc = + select_mg_client(&sagactx).await.map_err(|e| { + ActionError::action_failed(format!("select mg client (undo): {e}")) + })?; + + for peer in settings.bgp_peers { + let config = nexus + .bgp_config_get(&opctx, peer.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete bgp config: {e}")) + })?; + + mg_client + .inner + .delete_neighbor(&DeleteNeighborRequest { + asn: *config.asn, + addr: peer.addr.ip(), + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete neighbor: {e}")) + })?; + } + + Ok(()) +} + +async fn spa_undo_clear_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = + sagactx.lookup::>("original_switch_port_settings_id")?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = + nexus.switch_port_settings_get(&opctx, &NameOrId::Id(id)).await?; + + Ok(ensure_switch_port_bgp_settings(sagactx, settings).await?) +} + +async fn spa_clear_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let nexus = sagactx.user_data().nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + let mut config = + nexus.bootstore_network_config(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "read nexus bootstore network config: {e}" + )) + })?; + + let generation = nexus + .datastore() + .bump_bootstore_generation(&opctx, NETWORK_KEY.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "bump bootstore network generation number: {e}" + )) + })?; + + config.generation = generation as u64; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} + +async fn spa_undo_clear_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = sagactx + .lookup::>("original_switch_port_settings_id") + .map_err(|e| { + ActionError::action_failed(format!( + "original port settings id lookup: {e}" + )) + })?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = nexus + .switch_port_settings_get(&opctx, &NameOrId::Id(id)) + .await + .map_err(ActionError::action_failed)?; + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + // Read the current bootstore config, perform the update and write it back. + let mut config = read_bootstore_config(&sa).await?; + let update = bootstore_update( + &nexus, + &opctx, + params.switch_port_id, + ¶ms.port_name, + &settings, + ) + .await?; + apply_bootstore_update(&mut config, &update)?; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} diff --git a/nexus/src/app/sagas/switch_port_settings_update.rs b/nexus/src/app/sagas/switch_port_settings_update.rs deleted file mode 100644 index 23120bdbf4..0000000000 --- a/nexus/src/app/sagas/switch_port_settings_update.rs +++ /dev/null @@ -1,5 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// TODO https://github.com/oxidecomputer/omicron/issues/3002 diff --git a/nexus/src/app/sagas/volume_delete.rs b/nexus/src/app/sagas/volume_delete.rs index d6358d5435..43530e913c 100644 --- a/nexus/src/app/sagas/volume_delete.rs +++ b/nexus/src/app/sagas/volume_delete.rs @@ -155,15 +155,19 @@ async fn svd_delete_crucible_regions( sagactx.lookup::("crucible_resources_to_delete")?; // Send DELETE calls to the corresponding Crucible agents - let datasets_and_regions = match crucible_resources_to_delete { - CrucibleResources::V1(crucible_resources_to_delete) => { - crucible_resources_to_delete.datasets_and_regions - } - - CrucibleResources::V2(crucible_resources_to_delete) => { - crucible_resources_to_delete.datasets_and_regions - } - }; + let datasets_and_regions = osagactx + .datastore() + .regions_to_delete( + &crucible_resources_to_delete, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to get datasets_and_regions from crucible resources ({:?}): {:?}", + crucible_resources_to_delete, + e, + )) + })?; delete_crucible_regions(log, datasets_and_regions.clone()).await.map_err( |e| { @@ -208,31 +212,19 @@ async fn svd_delete_crucible_running_snapshots( sagactx.lookup::("crucible_resources_to_delete")?; // Send DELETE calls to the corresponding Crucible agents - let datasets_and_snapshots = match crucible_resources_to_delete { - CrucibleResources::V1(crucible_resources_to_delete) => { - crucible_resources_to_delete.datasets_and_snapshots - } - - CrucibleResources::V2(crucible_resources_to_delete) => { - let mut datasets_and_snapshots: Vec<_> = Vec::with_capacity( - crucible_resources_to_delete.snapshots_to_delete.len(), - ); - - for region_snapshot in - crucible_resources_to_delete.snapshots_to_delete - { - let dataset = osagactx - .datastore() - .dataset_get(region_snapshot.dataset_id) - .await - .map_err(ActionError::action_failed)?; - - datasets_and_snapshots.push((dataset, region_snapshot)); - } - - datasets_and_snapshots - } - }; + let datasets_and_snapshots = osagactx + .datastore() + .snapshots_to_delete( + &crucible_resources_to_delete, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to get datasets_and_snapshots from crucible resources ({:?}): {:?}", + crucible_resources_to_delete, + e, + )) + })?; delete_crucible_running_snapshots(log, datasets_and_snapshots.clone()) .await @@ -261,31 +253,19 @@ async fn svd_delete_crucible_snapshots( sagactx.lookup::("crucible_resources_to_delete")?; // Send DELETE calls to the corresponding Crucible agents - let datasets_and_snapshots = match crucible_resources_to_delete { - CrucibleResources::V1(crucible_resources_to_delete) => { - crucible_resources_to_delete.datasets_and_snapshots - } - - CrucibleResources::V2(crucible_resources_to_delete) => { - let mut datasets_and_snapshots: Vec<_> = Vec::with_capacity( - crucible_resources_to_delete.snapshots_to_delete.len(), - ); - - for region_snapshot in - crucible_resources_to_delete.snapshots_to_delete - { - let dataset = osagactx - .datastore() - .dataset_get(region_snapshot.dataset_id) - .await - .map_err(ActionError::action_failed)?; - - datasets_and_snapshots.push((dataset, region_snapshot)); - } - - datasets_and_snapshots - } - }; + let datasets_and_snapshots = osagactx + .datastore() + .snapshots_to_delete( + &crucible_resources_to_delete, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to get datasets_and_snapshots from crucible resources ({:?}): {:?}", + crucible_resources_to_delete, + e, + )) + })?; delete_crucible_snapshots(log, datasets_and_snapshots.clone()) .await @@ -308,56 +288,39 @@ async fn svd_delete_crucible_snapshot_records( let crucible_resources_to_delete = sagactx.lookup::("crucible_resources_to_delete")?; - match crucible_resources_to_delete { - CrucibleResources::V1(crucible_resources_to_delete) => { - // Remove DB records - for (_, region_snapshot) in - &crucible_resources_to_delete.datasets_and_snapshots - { - osagactx - .datastore() - .region_snapshot_remove( - region_snapshot.dataset_id, - region_snapshot.region_id, - region_snapshot.snapshot_id, - ) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "failed to region_snapshot_remove {} {} {}: {:?}", - region_snapshot.dataset_id, - region_snapshot.region_id, - region_snapshot.snapshot_id, - e, - )) - })?; - } - } - - CrucibleResources::V2(crucible_resources_to_delete) => { - // Remove DB records - for region_snapshot in - &crucible_resources_to_delete.snapshots_to_delete - { - osagactx - .datastore() - .region_snapshot_remove( - region_snapshot.dataset_id, - region_snapshot.region_id, - region_snapshot.snapshot_id, - ) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "failed to region_snapshot_remove {} {} {}: {:?}", - region_snapshot.dataset_id, - region_snapshot.region_id, - region_snapshot.snapshot_id, - e, - )) - })?; - } - } + // Remove DB records + let datasets_and_snapshots = osagactx + .datastore() + .snapshots_to_delete( + &crucible_resources_to_delete, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to get datasets_and_snapshots from crucible resources ({:?}): {:?}", + crucible_resources_to_delete, + e, + )) + })?; + + for (_, region_snapshot) in datasets_and_snapshots { + osagactx + .datastore() + .region_snapshot_remove( + region_snapshot.dataset_id, + region_snapshot.region_id, + region_snapshot.snapshot_id, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "failed to region_snapshot_remove {} {} {}: {:?}", + region_snapshot.dataset_id, + region_snapshot.region_id, + region_snapshot.snapshot_id, + e, + )) + })?; } Ok(()) diff --git a/nexus/src/app/switch_port.rs b/nexus/src/app/switch_port.rs index 996290b684..acc57459fd 100644 --- a/nexus/src/app/switch_port.rs +++ b/nexus/src/app/switch_port.rs @@ -2,31 +2,130 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//XXX +#![allow(unused_imports)] + use crate::app::sagas; use crate::external_api::params; use db::datastore::SwitchPortSettingsCombinedResult; +use dropshot::HttpError; +use http::StatusCode; +use ipnetwork::IpNetwork; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::authn; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use nexus_db_queries::db::datastore::UpdatePrecondition; use nexus_db_queries::db::model::{SwitchPort, SwitchPortSettings}; +use nexus_types::identity::Resource; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::{ self, CreateResult, DataPageParams, DeleteResult, ListResultVec, LookupResult, Name, NameOrId, UpdateResult, }; +use sled_agent_client::types::BgpConfig; +use sled_agent_client::types::BgpPeerConfig; +use sled_agent_client::types::{ + EarlyNetworkConfig, PortConfigV1, RackNetworkConfigV1, RouteConfig, +}; use std::sync::Arc; use uuid::Uuid; impl super::Nexus { - pub(crate) async fn switch_port_settings_create( - &self, + pub(crate) async fn switch_port_settings_post( + self: &Arc, opctx: &OpContext, params: params::SwitchPortSettingsCreate, ) -> CreateResult { opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; - self.db_datastore.switch_port_settings_create(opctx, ¶ms).await + + //TODO(ry) race conditions on exists check versus update/create. + // Normally I would use a DB lock here, but not sure what + // the Omicron way of doing things here is. + + match self + .db_datastore + .switch_port_settings_exist( + opctx, + params.identity.name.clone().into(), + ) + .await + { + Ok(id) => self.switch_port_settings_update(opctx, id, params).await, + Err(_) => { + self.switch_port_settings_create(opctx, params, None).await + } + } + } + + pub async fn switch_port_settings_create( + self: &Arc, + opctx: &OpContext, + params: params::SwitchPortSettingsCreate, + id: Option, + ) -> CreateResult { + self.db_datastore.switch_port_settings_create(opctx, ¶ms, id).await + } + + pub(crate) async fn switch_port_settings_update( + self: &Arc, + opctx: &OpContext, + switch_port_settings_id: Uuid, + new_settings: params::SwitchPortSettingsCreate, + ) -> CreateResult { + // delete old settings + self.switch_port_settings_delete( + opctx, + ¶ms::SwitchPortSettingsSelector { + port_settings: Some(NameOrId::Id(switch_port_settings_id)), + }, + ) + .await?; + + // create new settings + let result = self + .switch_port_settings_create( + opctx, + new_settings.clone(), + Some(switch_port_settings_id), + ) + .await?; + + // run the port settings apply saga for each port referencing the + // updated settings + + let ports = self + .db_datastore + .switch_ports_using_settings(opctx, switch_port_settings_id) + .await?; + + for (switch_port_id, switch_port_name) in ports.into_iter() { + let saga_params = sagas::switch_port_settings_apply::Params { + serialized_authn: authn::saga::Serialized::for_opctx(opctx), + switch_port_id, + switch_port_settings_id: result.settings.id(), + switch_port_name: switch_port_name.to_string(), + }; + + self.execute_saga::< + sagas::switch_port_settings_apply::SagaSwitchPortSettingsApply + >( + saga_params, + ) + .await + .map_err(|e| { + let msg = e.to_string(); + if msg.contains("bad request") { + //return HttpError::for_client_error(None, StatusCode::BAD_REQUEST, msg.to_string()) + external::Error::invalid_request(&msg.to_string()) + } else { + e + } + })?; + } + + Ok(result) } pub(crate) async fn switch_port_settings_delete( @@ -151,7 +250,9 @@ impl super::Nexus { switch_port_name: port.to_string(), }; - self.execute_saga::( + self.execute_saga::< + sagas::switch_port_settings_apply::SagaSwitchPortSettingsApply + >( saga_params, ) .await?; @@ -215,4 +316,25 @@ impl super::Nexus { Ok(()) } + + // TODO it would likely be better to do this as a one shot db query. + pub(crate) async fn active_port_settings( + &self, + opctx: &OpContext, + ) -> LookupResult> { + let mut ports = Vec::new(); + let port_list = + self.switch_port_list(opctx, &DataPageParams::max_page()).await?; + + for p in port_list { + if let Some(id) = p.port_settings_id { + ports.push(( + p.clone(), + self.switch_port_settings_get(opctx, &id.into()).await?, + )); + } + } + + LookupResult::Ok(ports) + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 1fddfba85b..48be2de6b0 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -63,6 +63,11 @@ use omicron_common::api::external::http_pagination::ScanParams; use omicron_common::api::external::AddressLot; use omicron_common::api::external::AddressLotBlock; use omicron_common::api::external::AddressLotCreateResponse; +use omicron_common::api::external::BgpAnnounceSet; +use omicron_common::api::external::BgpAnnouncement; +use omicron_common::api::external::BgpConfig; +use omicron_common::api::external::BgpImportedRouteIpv4; +use omicron_common::api::external::BgpPeerStatus; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Disk; use omicron_common::api::external::Error; @@ -250,6 +255,15 @@ pub(crate) fn external_api() -> NexusApiDescription { api.register(networking_switch_port_apply_settings)?; api.register(networking_switch_port_clear_settings)?; + api.register(networking_bgp_config_create)?; + api.register(networking_bgp_config_list)?; + api.register(networking_bgp_status)?; + api.register(networking_bgp_imported_routes_ipv4)?; + api.register(networking_bgp_config_delete)?; + api.register(networking_bgp_announce_set_create)?; + api.register(networking_bgp_announce_set_list)?; + api.register(networking_bgp_announce_set_delete)?; + // Fleet-wide API operations api.register(silo_list)?; api.register(silo_create)?; @@ -2595,7 +2609,7 @@ async fn networking_loopback_address_delete( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } -/// Get loopback addresses, optionally filtering by id +/// List loopback addresses #[endpoint { method = GET, path = "/v1/system/networking/loopback-address", @@ -2642,7 +2656,7 @@ async fn networking_switch_port_settings_create( let nexus = &apictx.nexus; let params = new_settings.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus.switch_port_settings_create(&opctx, params).await?; + let result = nexus.switch_port_settings_post(&opctx, params).await?; let settings: SwitchPortSettingsView = result.into(); Ok(HttpResponseCreated(settings)) @@ -2810,6 +2824,193 @@ async fn networking_switch_port_clear_settings( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } +/// Create a new BGP configuration +#[endpoint { + method = POST, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], +}] +async fn networking_bgp_config_create( + rqctx: RequestContext>, + config: TypedBody, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let config = config.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.bgp_config_set(&opctx, &config).await?; + Ok(HttpResponseCreated::(result.into())) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// List BGP configurations +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], +}] +async fn networking_bgp_config_list( + rqctx: RequestContext>, + query_params: Query>, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let configs = nexus + .bgp_config_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + configs, + &marker_for_name_or_id, + )?)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +//TODO pagination? the normal by-name/by-id stuff does not work here +/// Get BGP peer status +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp-status", + tags = ["system/networking"], +}] +async fn networking_bgp_status( + rqctx: RequestContext>, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let handler = async { + let nexus = &apictx.nexus; + let result = nexus.bgp_peer_status(&opctx).await?; + Ok(HttpResponseOk(result)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +//TODO pagination? the normal by-name/by-id stuff does not work here +/// Get imported IPv4 BGP routes +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp-routes-ipv4", + tags = ["system/networking"], +}] +async fn networking_bgp_imported_routes_ipv4( + rqctx: RequestContext>, + query_params: Query, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let handler = async { + let nexus = &apictx.nexus; + let sel = query_params.into_inner(); + let result = nexus.bgp_imported_routes_ipv4(&opctx, &sel).await?; + Ok(HttpResponseOk(result)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Delete a BGP configuration +#[endpoint { + method = DELETE, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], +}] +async fn networking_bgp_config_delete( + rqctx: RequestContext>, + sel: Query, +) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let sel = sel.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + nexus.bgp_config_delete(&opctx, &sel).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Create a new BGP announce set +#[endpoint { + method = POST, + path = "/v1/system/networking/bgp-announce", + tags = ["system/networking"], +}] +async fn networking_bgp_announce_set_create( + rqctx: RequestContext>, + config: TypedBody, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let config = config.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.bgp_create_announce_set(&opctx, &config).await?; + Ok(HttpResponseCreated::(result.0.into())) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +//TODO pagination? the normal by-name/by-id stuff does not work here +/// Get originated routes for a BGP configuration +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp-announce", + tags = ["system/networking"], +}] +async fn networking_bgp_announce_set_list( + rqctx: RequestContext>, + query_params: Query, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let sel = query_params.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus + .bgp_announce_list(&opctx, &sel) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + Ok(HttpResponseOk(result)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Delete a BGP announce set +#[endpoint { + method = DELETE, + path = "/v1/system/networking/bgp-announce", + tags = ["system/networking"], +}] +async fn networking_bgp_announce_set_delete( + rqctx: RequestContext>, + selector: Query, +) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let sel = selector.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + nexus.bgp_delete_announce_set(&opctx, &sel).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + // Images /// List images diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index 586c828683..01aca36e1d 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -26,17 +26,18 @@ pub use app::test_interfaces::TestInterfaces; pub use app::Nexus; pub use config::Config; use context::ServerContext; +use dropshot::ConfigDropshot; use external_api::http_entrypoints::external_api; use internal_api::http_entrypoints::internal_api; use nexus_types::internal_api::params::ServiceKind; use omicron_common::address::IpRange; use omicron_common::api::internal::shared::{ - ExternalPortDiscovery, SwitchLocation, + ExternalPortDiscovery, RackNetworkConfig, SwitchLocation, }; use omicron_common::FileKv; use slog::Logger; use std::collections::HashMap; -use std::net::{SocketAddr, SocketAddrV6}; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -131,6 +132,23 @@ impl Server { .nexus .external_tls_config(config.deployment.dropshot_external.tls) .await; + + // We launch two dropshot servers providing the external API: one as + // configured (which is accessible from the customer network), and one + // that matches the configuration except listens on the same address + // (but a different port) as the `internal` server. The latter is + // available for proxied connections via the tech port in the event the + // rack has lost connectivity (see RFD 431). + let techport_server_bind_addr = { + let mut addr = http_server_internal.local_addr(); + addr.set_port(config.deployment.techport_external_server_port); + addr + }; + let techport_server_config = ConfigDropshot { + bind_address: techport_server_bind_addr, + ..config.deployment.dropshot_external.dropshot.clone() + }; + let http_server_external = { let server_starter_external = dropshot::HttpServerStarter::new_with_tls( @@ -138,16 +156,35 @@ impl Server { external_api(), Arc::clone(&apictx), &log.new(o!("component" => "dropshot_external")), - tls_config.map(dropshot::ConfigTls::Dynamic), + tls_config.clone().map(dropshot::ConfigTls::Dynamic), ) .map_err(|error| { format!("initializing external server: {}", error) })?; server_starter_external.start() }; + let http_server_techport_external = { + let server_starter_external_techport = + dropshot::HttpServerStarter::new_with_tls( + &techport_server_config, + external_api(), + Arc::clone(&apictx), + &log.new(o!("component" => "dropshot_external_techport")), + tls_config.map(dropshot::ConfigTls::Dynamic), + ) + .map_err(|error| { + format!("initializing external techport server: {}", error) + })?; + server_starter_external_techport.start() + }; + apictx .nexus - .set_servers(http_server_external, http_server_internal) + .set_servers( + http_server_external, + http_server_techport_external, + http_server_internal, + ) .await; let server = Server { apictx: apictx.clone() }; Ok(server) @@ -252,7 +289,13 @@ impl nexus_test_interface::NexusServer for Server { vec!["qsfp0".parse().unwrap()], )]), ), - rack_network_config: None, + rack_network_config: Some(RackNetworkConfig { + rack_subnet: "fd00:1122:3344:01::/56".parse().unwrap(), + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports: Vec::new(), + bgp: Vec::new(), + }), }, ) .await diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 4ac64082b9..647232031d 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -59,6 +59,7 @@ pub const RACK_UUID: &str = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"; pub const SWITCH_UUID: &str = "dae4e1f1-410e-4314-bff1-fec0504be07e"; pub const OXIMETER_UUID: &str = "39e6175b-4df2-4730-b11d-cbc1e60a2e78"; pub const PRODUCER_UUID: &str = "a6458b7d-87c3-4483-be96-854d814c20de"; +pub const RACK_SUBNET: &str = "fd00:1122:3344:01::/56"; /// The reported amount of hardware threads for an emulated sled agent. pub const TEST_HARDWARE_THREADS: u32 = 16; @@ -88,6 +89,7 @@ pub struct ControlPlaneTestContext { pub producer: ProducerServer, pub gateway: GatewayTestContext, pub dendrite: HashMap, + pub mgd: HashMap, pub external_dns_zone_name: String, pub external_dns: dns_server::TransientServer, pub internal_dns: dns_server::TransientServer, @@ -111,6 +113,9 @@ impl ControlPlaneTestContext { for (_, mut dendrite) in self.dendrite { dendrite.cleanup().await.unwrap(); } + for (_, mut mgd) in self.mgd { + mgd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } } @@ -242,6 +247,7 @@ pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> { pub producer: Option, pub gateway: Option, pub dendrite: HashMap, + pub mgd: HashMap, // NOTE: Only exists after starting Nexus, until external Nexus is // initialized. @@ -281,6 +287,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { producer: None, gateway: None, dendrite: HashMap::new(), + mgd: HashMap::new(), nexus_internal: None, nexus_internal_addr: None, external_dns_zone_name: None, @@ -436,6 +443,32 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { ); } + pub async fn start_mgd(&mut self, switch_location: SwitchLocation) { + let log = &self.logctx.log; + debug!(log, "Starting mgd for {switch_location}"); + + // Set up an instance of mgd + let mgd = dev::maghemite::MgdInstance::start(0).await.unwrap(); + let port = mgd.port; + self.mgd.insert(switch_location, mgd); + let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); + + debug!(log, "mgd port is {port}"); + + let config = omicron_common::nexus_config::MgdConfig { + address: std::net::SocketAddr::V6(address), + }; + self.config.pkg.mgd.insert(switch_location, config); + + let sled_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap(); + self.rack_init_builder.add_service( + address, + ServiceKind::Mgd, + internal_dns::ServiceName::Mgd, + sled_id, + ); + } + pub async fn start_oximeter(&mut self) { let log = &self.logctx.log; debug!(log, "Starting Oximeter"); @@ -566,8 +599,11 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { &format!("http://{}", internal_dns_address), log.clone(), ); + let dns_config = self.rack_init_builder.internal_dns_config.clone().build(); + + slog::info!(log, "DNS population: {:#?}", dns_config); dns_config_client.dns_config_put(&dns_config).await.expect( "Failed to send initial DNS records to internal DNS server", ); @@ -707,6 +743,25 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { ); } + pub async fn scrimlet_dns_setup(&mut self) { + let sled_agent = self + .sled_agent + .as_ref() + .expect("Cannot set up scrimlet DNS without sled agent"); + + let sa = match sled_agent.http_server.local_addr() { + SocketAddr::V6(sa) => sa, + SocketAddr::V4(_) => panic!("expected SocketAddrV6 for sled agent"), + }; + + for loc in [SwitchLocation::Switch0, SwitchLocation::Switch1] { + self.rack_init_builder + .internal_dns_config + .host_scrimlet(loc, sa) + .expect("add switch0 scrimlet dns entry"); + } + } + // Set up an external DNS server. pub async fn start_external_dns(&mut self) { let log = self.logctx.log.new(o!("component" => "external_dns_server")); @@ -781,6 +836,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { logctx: self.logctx, gateway: self.gateway.unwrap(), dendrite: self.dendrite, + mgd: self.mgd, external_dns_zone_name: self.external_dns_zone_name.unwrap(), external_dns: self.external_dns.unwrap(), internal_dns: self.internal_dns.unwrap(), @@ -814,6 +870,9 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { for (_, mut dendrite) in self.dendrite { dendrite.cleanup().await.unwrap(); } + for (_, mut mgd) in self.mgd { + mgd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } } @@ -905,11 +964,14 @@ async fn setup_with_config_impl( builder.start_gateway().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.start_internal_dns().await; builder.start_external_dns().await; builder.start_nexus_internal().await; builder.start_sled(sim_mode).await; builder.start_crucible_pantry().await; + builder.scrimlet_dns_setup().await; // Give Nexus necessary information to find the Crucible Pantry let dns_config = builder.populate_internal_dns().await; diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index 3629ae9cb2..54f7e03eef 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -39,6 +39,7 @@ max_vpc_ipv4_subnet_prefix = 29 # NOTE: The test suite always overrides this. id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c" rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc" +techport_external_server_port = 0 # Nexus may need to resolve external hosts (e.g. to grab IdP metadata). # These are the DNS servers it should use. diff --git a/nexus/tests/integration_tests/address_lots.rs b/nexus/tests/integration_tests/address_lots.rs index b4659daa62..40c8865929 100644 --- a/nexus/tests/integration_tests/address_lots.rs +++ b/nexus/tests/integration_tests/address_lots.rs @@ -27,8 +27,8 @@ type ControlPlaneTestContext = async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { let client = &ctx.external_client; - // Verify there are no lots - let lots = NexusRequest::iter_collection_authn::( + // Verify there is only one system lot + let lots = NexusRequest::iter_collection_authn::( client, "/v1/system/networking/address-lot", "", @@ -37,7 +37,7 @@ async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { .await .expect("Failed to list address lots") .all_items; - assert_eq!(lots.len(), 0, "Expected no lots"); + assert_eq!(lots.len(), 1, "Expected one lot"); // Create a lot let params = AddressLotCreate { @@ -111,8 +111,8 @@ async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { .expect("Failed to list address lots") .all_items; - assert_eq!(lots.len(), 1, "Expected 1 lot"); - assert_eq!(lots[0], address_lot); + assert_eq!(lots.len(), 2, "Expected 2 lots"); + assert_eq!(lots[1], address_lot); // Verify there are lot blocks let blist = NexusRequest::iter_collection_authn::( diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index 71a3977192..a5a8339c34 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -1672,6 +1672,84 @@ async fn test_disk_create_for_importing(cptestctx: &ControlPlaneTestContext) { disks_eq(&disks[0], &disk); } +#[nexus_test] +async fn test_project_delete_disk_no_auth_idempotent( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + DiskTest::new(&cptestctx).await; + create_org_and_project(client).await; + + // Create a disk + let disks_url = get_disks_url(); + + let new_disk = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: DISK_NAME.parse().unwrap(), + description: String::from("sells rainsticks"), + }, + disk_source: params::DiskSource::Blank { + block_size: params::BlockSize::try_from(512).unwrap(), + }, + size: ByteCount::from_gibibytes_u32(1), + }; + + NexusRequest::new( + RequestBuilder::new(client, Method::POST, &disks_url) + .body(Some(&new_disk)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body::() + .unwrap(); + + let disk_url = get_disk_url(DISK_NAME); + let disk = disk_get(&client, &disk_url).await; + assert_eq!(disk.state, DiskState::Detached); + + // Call project_delete_disk_no_auth twice, ensuring that the disk is either + // there before deleting and not afterwards. + + let nexus = &cptestctx.server.apictx().nexus; + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + let (.., db_disk) = LookupPath::new(&opctx, &datastore) + .disk_id(disk.identity.id) + .fetch() + .await + .unwrap(); + + assert_eq!(db_disk.id(), disk.identity.id); + + datastore + .project_delete_disk_no_auth( + &disk.identity.id, + &[DiskState::Detached, DiskState::Faulted], + ) + .await + .unwrap(); + + let r = LookupPath::new(&opctx, &datastore) + .disk_id(disk.identity.id) + .fetch() + .await; + + assert!(r.is_err()); + + datastore + .project_delete_disk_no_auth( + &disk.identity.id, + &[DiskState::Detached, DiskState::Faulted], + ) + .await + .unwrap(); +} + async fn disk_get(client: &ClientTestContext, disk_url: &str) -> Disk { NexusRequest::object_get(client, disk_url) .authn_as(AuthnMode::PrivilegedUser) diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index e9ae11c21f..8fba22fb2f 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -420,6 +420,40 @@ lazy_static! { }; } +lazy_static! { + pub static ref DEMO_BGP_CONFIG_CREATE_URL: String = + format!("/v1/system/networking/bgp?name_or_id=as47"); + pub static ref DEMO_BGP_CONFIG: params::BgpConfigCreate = + params::BgpConfigCreate { + identity: IdentityMetadataCreateParams { + name: "as47".parse().unwrap(), + description: "BGP config for AS47".into(), + }, + bgp_announce_set_id: NameOrId::Name("instances".parse().unwrap()), + asn: 47, + vrf: None, + }; + pub static ref DEMO_BGP_ANNOUNCE_SET_URL: String = + format!("/v1/system/networking/bgp-announce?name_or_id=a-bag-of-addrs"); + pub static ref DEMO_BGP_ANNOUNCE: params::BgpAnnounceSetCreate = + params::BgpAnnounceSetCreate { + identity: IdentityMetadataCreateParams { + name: "a-bag-of-addrs".parse().unwrap(), + description: "a bag of addrs".into(), + }, + announcement: vec![params::BgpAnnouncementCreate { + address_lot_block: NameOrId::Name( + "some-block".parse().unwrap(), + ), + network: "10.0.0.0/16".parse().unwrap(), + }], + }; + pub static ref DEMO_BGP_STATUS_URL: String = + format!("/v1/system/networking/bgp-status"); + pub static ref DEMO_BGP_ROUTES_IPV4_URL: String = + format!("/v1/system/networking/bgp-routes-ipv4?asn=47"); +} + lazy_static! { // Project Images pub static ref DEMO_IMAGE_NAME: Name = "demo-image".parse().unwrap(); @@ -1876,5 +1910,48 @@ lazy_static! { AllowedMethod::GetNonexistent ], }, + VerifyEndpoint { + url: &DEMO_BGP_CONFIG_CREATE_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_BGP_CONFIG).unwrap(), + ), + AllowedMethod::Get, + AllowedMethod::Delete + ], + }, + + VerifyEndpoint { + url: &DEMO_BGP_ANNOUNCE_SET_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_BGP_ANNOUNCE).unwrap(), + ), + AllowedMethod::GetNonexistent, + AllowedMethod::Delete + ], + }, + + VerifyEndpoint { + url: &DEMO_BGP_STATUS_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + ], + }, + + VerifyEndpoint { + url: &DEMO_BGP_ROUTES_IPV4_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + ], + } ]; } diff --git a/nexus/tests/integration_tests/images.rs b/nexus/tests/integration_tests/images.rs index 84a8a1f50f..c3db9e8f13 100644 --- a/nexus/tests/integration_tests/images.rs +++ b/nexus/tests/integration_tests/images.rs @@ -7,15 +7,21 @@ use dropshot::ResultsPage; use http::method::Method; use http::StatusCode; +use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; +use nexus_db_queries::db::fixed_data::silo_user::USER_TEST_UNPRIVILEGED; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils::resource_helpers::grant_iam; use nexus_test_utils::resource_helpers::DiskTest; use nexus_test_utils_macros::nexus_test; -use omicron_common::api::external::Disk; - +use nexus_types::external_api::shared::ProjectRole; +use nexus_types::external_api::shared::SiloRole; use nexus_types::external_api::{params, views}; +use nexus_types::identity::Asset; +use nexus_types::identity::Resource; +use omicron_common::api::external::Disk; use omicron_common::api::external::{ByteCount, IdentityMetadataCreateParams}; use httptest::{matchers::*, responders::*, Expectation, ServerBuilder}; @@ -709,3 +715,121 @@ async fn test_image_from_other_project_snapshot_fails( .unwrap(); assert_eq!(error.message, "snapshot does not belong to this project"); } + +#[nexus_test] +async fn test_image_deletion_permissions(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + DiskTest::new(&cptestctx).await; + + // Create a project + + create_project(client, PROJECT_NAME).await; + + // Grant the unprivileged user viewer on the silo and admin on that project + + let silo_url = format!("/v1/system/silos/{}", DEFAULT_SILO.id()); + grant_iam( + client, + &silo_url, + SiloRole::Viewer, + USER_TEST_UNPRIVILEGED.id(), + AuthnMode::PrivilegedUser, + ) + .await; + + let project_url = format!("/v1/projects/{}", PROJECT_NAME); + grant_iam( + client, + &project_url, + ProjectRole::Admin, + USER_TEST_UNPRIVILEGED.id(), + AuthnMode::PrivilegedUser, + ) + .await; + + // Create an image in the default silo using the privileged user + + let server = ServerBuilder::new().run().unwrap(); + server.expect( + Expectation::matching(request::method_path("HEAD", "/image.raw")) + .times(1..) + .respond_with( + status_code(200).append_header( + "Content-Length", + format!("{}", 4096 * 1000), + ), + ), + ); + + let silo_images_url = "/v1/images"; + let images_url = get_project_images_url(PROJECT_NAME); + + let image_create_params = get_image_create(params::ImageSource::Url { + url: server.url("/image.raw").to_string(), + block_size: BLOCK_SIZE, + }); + + let image = + NexusRequest::objects_post(client, &images_url, &image_create_params) + .authn_as(AuthnMode::PrivilegedUser) + .execute_and_parse_unwrap::() + .await; + + let image_id = image.identity.id; + + // promote the image to the silo + + let promote_url = format!("/v1/images/{}/promote", image_id); + NexusRequest::new( + RequestBuilder::new(client, http::Method::POST, &promote_url) + .expect_status(Some(http::StatusCode::ACCEPTED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute_and_parse_unwrap::() + .await; + + let silo_images = NexusRequest::object_get(client, &silo_images_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute_and_parse_unwrap::>() + .await + .items; + + assert_eq!(silo_images.len(), 1); + assert_eq!(silo_images[0].identity.name, "alpine-edge"); + + // the unprivileged user should not be able to delete that image + + let image_url = format!("/v1/images/{}", image_id); + NexusRequest::new( + RequestBuilder::new(client, http::Method::DELETE, &image_url) + .expect_status(Some(http::StatusCode::FORBIDDEN)), + ) + .authn_as(AuthnMode::UnprivilegedUser) + .execute() + .await + .expect("should not be able to delete silo image as unpriv user!"); + + // Demote that image + + let demote_url = + format!("/v1/images/{}/demote?project={}", image_id, PROJECT_NAME); + NexusRequest::new( + RequestBuilder::new(client, http::Method::POST, &demote_url) + .expect_status(Some(http::StatusCode::ACCEPTED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute_and_parse_unwrap::() + .await; + + // now the unpriviledged user should be able to delete that image + + let image_url = format!("/v1/images/{}", image_id); + NexusRequest::new( + RequestBuilder::new(client, http::Method::DELETE, &image_url) + .expect_status(Some(http::StatusCode::NO_CONTENT)), + ) + .authn_as(AuthnMode::UnprivilegedUser) + .execute() + .await + .expect("should be able to delete project image as unpriv user!"); +} diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index 2d4c76dc99..43a4ac8f2e 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -29,6 +29,8 @@ async fn test_nexus_boots_before_cockroach() { builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.start_internal_dns().await; builder.start_external_dns().await; @@ -144,6 +146,11 @@ async fn test_nexus_boots_before_dendrite() { builder.start_dendrite(SwitchLocation::Switch1).await; info!(log, "Started Dendrite"); + info!(log, "Starting mgd"); + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; + info!(log, "Started mgd"); + info!(log, "Populating internal DNS records"); builder.populate_internal_dns().await; info!(log, "Populated internal DNS records"); @@ -166,6 +173,8 @@ async fn nexus_schema_test_setup( builder.start_external_dns().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.populate_internal_dns().await; } diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 9208e21652..ea633be9dc 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -942,7 +942,7 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { assert_metrics(cptestctx, project_id, 0, 0, 0).await; - // Create an instance. + // Create and start an instance. let instance_name = "just-rainsticks"; create_instance(client, PROJECT_NAME, instance_name).await; let virtual_provisioning_collection = datastore @@ -955,27 +955,22 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { ByteCount::from_gibibytes_u32(1), ); - // Stop the instance + // Stop the instance. This should cause the relevant resources to be + // deprovisioned. let instance = instance_post(&client, instance_name, InstanceOp::Stop).await; instance_simulate(nexus, &instance.identity.id).await; let instance = instance_get(&client, &get_instance_url(&instance_name)).await; assert_eq!(instance.runtime.run_state, InstanceState::Stopped); - // NOTE: I think it's arguably "more correct" to identify that the - // number of CPUs being used by guests at this point is actually "0", - // not "4", because the instance is stopped (same re: RAM usage). - // - // However, for implementation reasons, this is complicated (we have a - // tendency to update the runtime without checking the prior state, which - // makes edge-triggered behavior trickier to notice). + let virtual_provisioning_collection = datastore .virtual_provisioning_collection_get(&opctx, project_id) .await .unwrap(); - let expected_cpus = 4; + let expected_cpus = 0; let expected_ram = - i64::try_from(ByteCount::from_gibibytes_u32(1).to_bytes()).unwrap(); + i64::try_from(ByteCount::from_gibibytes_u32(0).to_bytes()).unwrap(); assert_eq!(virtual_provisioning_collection.cpus_provisioned, expected_cpus); assert_eq!( i64::from(virtual_provisioning_collection.ram_provisioned.0), @@ -983,7 +978,7 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { ); assert_metrics(cptestctx, project_id, 0, expected_cpus, expected_ram).await; - // Stop the instance + // Delete the instance. NexusRequest::object_delete(client, &get_instance_url(&instance_name)) .authn_as(AuthnMode::PrivilegedUser) .execute() @@ -999,6 +994,130 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { assert_metrics(cptestctx, project_id, 0, 0, 0).await; } +#[nexus_test] +async fn test_instance_metrics_with_migration( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + let instance_name = "bird-ecology"; + + // Create a second sled to migrate to/from. + let default_sled_id: Uuid = + nexus_test_utils::SLED_AGENT_UUID.parse().unwrap(); + let update_dir = Utf8Path::new("/should/be/unused"); + let other_sled_id = Uuid::new_v4(); + let _other_sa = nexus_test_utils::start_sled_agent( + cptestctx.logctx.log.new(o!("sled_id" => other_sled_id.to_string())), + cptestctx.server.get_http_server_internal_address().await, + other_sled_id, + &update_dir, + sim::SimMode::Explicit, + ) + .await + .unwrap(); + + let project_id = create_org_and_project(&client).await; + let instance_url = get_instance_url(instance_name); + + // Explicitly create an instance with no disks. Simulated sled agent assumes + // that disks are co-located with their instances. + let instance = nexus_test_utils::resource_helpers::create_instance_with( + client, + PROJECT_NAME, + instance_name, + ¶ms::InstanceNetworkInterfaceAttachment::Default, + Vec::::new(), + Vec::::new(), + ) + .await; + let instance_id = instance.identity.id; + + // Poke the instance into an active state. + instance_simulate(nexus, &instance_id).await; + let instance_next = instance_get(&client, &instance_url).await; + assert_eq!(instance_next.runtime.run_state, InstanceState::Running); + + // The instance should be provisioned while it's in the running state. + let nexus = &apictx.nexus; + let datastore = nexus.datastore(); + let check_provisioning_state = |cpus: i64, mem_gib: u32| async move { + let opctx = OpContext::for_tests( + cptestctx.logctx.log.new(o!()), + datastore.clone(), + ); + let virtual_provisioning_collection = datastore + .virtual_provisioning_collection_get(&opctx, project_id) + .await + .unwrap(); + assert_eq!( + virtual_provisioning_collection.cpus_provisioned, + cpus.clone() + ); + assert_eq!( + virtual_provisioning_collection.ram_provisioned.0, + ByteCount::from_gibibytes_u32(mem_gib) + ); + }; + + check_provisioning_state(4, 1).await; + + // Request migration to the other sled. This reserves resources on the + // target sled, but shouldn't change the virtual provisioning counters. + let original_sled = nexus + .instance_sled_id(&instance_id) + .await + .unwrap() + .expect("running instance should have a sled"); + + let dst_sled_id = if original_sled == default_sled_id { + other_sled_id + } else { + default_sled_id + }; + + let migrate_url = + format!("/v1/instances/{}/migrate", &instance_id.to_string()); + let _ = NexusRequest::new( + RequestBuilder::new(client, Method::POST, &migrate_url) + .body(Some(¶ms::InstanceMigrate { dst_sled_id })) + .expect_status(Some(StatusCode::OK)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body::() + .unwrap(); + + check_provisioning_state(4, 1).await; + + // Complete migration on the target. Simulated migrations always succeed. + // After this the instance should be running and should continue to appear + // to be provisioned. + instance_simulate_on_sled(cptestctx, nexus, dst_sled_id, instance_id).await; + let instance = instance_get(&client, &instance_url).await; + assert_eq!(instance.runtime.run_state, InstanceState::Running); + + check_provisioning_state(4, 1).await; + + // Now stop the instance. This should retire the instance's active Propolis + // and cause the virtual provisioning charges to be released. Note that + // the source sled still has an active resource charge for the source + // instance (whose demise wasn't simulated here), but this is intentionally + // not reflected in the virtual provisioning counters (which reflect the + // logical states of instances ignoring migration). + let instance = + instance_post(&client, instance_name, InstanceOp::Stop).await; + instance_simulate(nexus, &instance.identity.id).await; + let instance = + instance_get(&client, &get_instance_url(&instance_name)).await; + assert_eq!(instance.runtime.run_state, InstanceState::Stopped); + + check_provisioning_state(0, 0).await; +} + #[nexus_test] async fn test_instances_create_stopped_start( cptestctx: &ControlPlaneTestContext, diff --git a/nexus/tests/integration_tests/projects.rs b/nexus/tests/integration_tests/projects.rs index f974e85dc4..24b2721a1d 100644 --- a/nexus/tests/integration_tests/projects.rs +++ b/nexus/tests/integration_tests/projects.rs @@ -245,32 +245,24 @@ async fn test_project_deletion_with_image(cptestctx: &ControlPlaneTestContext) { delete_project_expect_fail(&url, &client).await, ); - // TODO: finish test once image delete is implemented. Image create works - // and project delete with image fails as expected, but image delete is not - // implemented yet, so we can't show that project delete works after image - // delete. let image_url = format!("/v1/images/{}", image.identity.id); - NexusRequest::expect_failure_with_body( - client, - StatusCode::INTERNAL_SERVER_ERROR, - Method::DELETE, - &image_url, - &image_create_params, + NexusRequest::object_delete(&client, &image_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete image"); + + // Expect that trying to GET the image results in a 404 + NexusRequest::new( + RequestBuilder::new(&client, http::Method::GET, &image_url) + .expect_status(Some(http::StatusCode::NOT_FOUND)), ) .authn_as(AuthnMode::PrivilegedUser) .execute() .await - .unwrap(); + .expect("GET of a deleted image did not return 404"); - // TODO: delete the image - // NexusRequest::object_delete(&client, &image_url) - // .authn_as(AuthnMode::PrivilegedUser) - // .execute() - // .await - // .expect("failed to delete image"); - - // TODO: now delete project works - // delete_project(&url, &client).await; + delete_project(&url, &client).await; } #[nexus_test] diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 6d2595b561..d79dd09fc1 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -58,6 +58,8 @@ async fn test_setup<'a>( builder.start_external_dns().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.populate_internal_dns().await; builder } @@ -87,6 +89,7 @@ async fn apply_update_as_transaction( match apply_update_as_transaction_inner(client, sql).await { Ok(()) => break, Err(err) => { + warn!(log, "Failed to apply update as transaction"; "err" => err.to_string()); client .batch_execute("ROLLBACK;") .await @@ -109,7 +112,9 @@ async fn apply_update( version: &str, times_to_apply: usize, ) { - info!(log, "Performing upgrade to {version}"); + let log = log.new(o!("target version" => version.to_string())); + info!(log, "Performing upgrade"); + let client = crdb.connect().await.expect("failed to connect"); // We skip this for the earliest supported version because these tables @@ -124,11 +129,15 @@ async fn apply_update( } let target_dir = Utf8PathBuf::from(SCHEMA_DIR).join(version); - let sqls = all_sql_for_version_migration(&target_dir).await.unwrap(); + let schema_change = + all_sql_for_version_migration(&target_dir).await.unwrap(); for _ in 0..times_to_apply { - for sql in sqls.iter() { - apply_update_as_transaction(log, &client, sql).await; + for nexus_db_queries::db::datastore::SchemaUpgradeStep { path, sql } in + &schema_change.steps + { + info!(log, "Applying sql schema upgrade step"; "path" => path.to_string()); + apply_update_as_transaction(&log, &client, sql).await; } } @@ -253,7 +262,35 @@ impl<'a> From<&'a [&'static str]> for ColumnSelector<'a> { } } -async fn query_crdb_for_rows_of_strings( +async fn crdb_show_constraints( + crdb: &CockroachInstance, + table: &str, +) -> Vec { + let client = crdb.connect().await.expect("failed to connect"); + + let sql = format!("SHOW CONSTRAINTS FROM {table}"); + let rows = client + .query(&sql, &[]) + .await + .unwrap_or_else(|_| panic!("failed to query {table}")); + client.cleanup().await.expect("cleaning up after wipe"); + + let mut result = vec![]; + for row in rows { + let mut row_result = Row::new(); + for i in 0..row.len() { + let column_name = row.columns()[i].name(); + row_result.values.push(NamedSqlValue { + column: column_name.to_string(), + value: row.get(i), + }); + } + result.push(row_result); + } + result +} + +async fn crdb_select( crdb: &CockroachInstance, columns: ColumnSelector<'_>, table: &str, @@ -453,22 +490,16 @@ async fn versions_have_idempotent_up() { logctx.cleanup_successful(); } -const COLUMNS: [&'static str; 6] = [ +const COLUMNS: [&'static str; 7] = [ "table_catalog", "table_schema", "table_name", "column_name", "column_default", + "is_nullable", "data_type", ]; -const CHECK_CONSTRAINTS: [&'static str; 4] = [ - "constraint_catalog", - "constraint_schema", - "constraint_name", - "check_clause", -]; - const CONSTRAINT_COLUMN_USAGE: [&'static str; 7] = [ "table_catalog", "table_schema", @@ -538,22 +569,9 @@ const PG_INDEXES: [&'static str; 5] = const TABLES: [&'static str; 4] = ["table_catalog", "table_schema", "table_name", "table_type"]; -const TABLE_CONSTRAINTS: [&'static str; 9] = [ - "constraint_catalog", - "constraint_schema", - "constraint_name", - "table_catalog", - "table_schema", - "table_name", - "constraint_type", - "is_deferrable", - "initially_deferred", -]; - #[derive(Eq, PartialEq, Debug)] struct InformationSchema { columns: Vec, - check_constraints: Vec, constraint_column_usage: Vec, key_column_usage: Vec, referential_constraints: Vec, @@ -562,7 +580,7 @@ struct InformationSchema { sequences: Vec, pg_indexes: Vec, tables: Vec, - table_constraints: Vec, + table_constraints: BTreeMap>, } impl InformationSchema { @@ -576,10 +594,6 @@ impl InformationSchema { self.table_constraints, other.table_constraints ); - similar_asserts::assert_eq!( - self.check_constraints, - other.check_constraints - ); similar_asserts::assert_eq!( self.constraint_column_usage, other.constraint_column_usage @@ -602,7 +616,7 @@ impl InformationSchema { // https://www.cockroachlabs.com/docs/v23.1/information-schema // // For details on each of these tables. - let columns = query_crdb_for_rows_of_strings( + let columns = crdb_select( crdb, COLUMNS.as_slice().into(), "information_schema.columns", @@ -610,15 +624,7 @@ impl InformationSchema { ) .await; - let check_constraints = query_crdb_for_rows_of_strings( - crdb, - CHECK_CONSTRAINTS.as_slice().into(), - "information_schema.check_constraints", - None, - ) - .await; - - let constraint_column_usage = query_crdb_for_rows_of_strings( + let constraint_column_usage = crdb_select( crdb, CONSTRAINT_COLUMN_USAGE.as_slice().into(), "information_schema.constraint_column_usage", @@ -626,7 +632,7 @@ impl InformationSchema { ) .await; - let key_column_usage = query_crdb_for_rows_of_strings( + let key_column_usage = crdb_select( crdb, KEY_COLUMN_USAGE.as_slice().into(), "information_schema.key_column_usage", @@ -634,7 +640,7 @@ impl InformationSchema { ) .await; - let referential_constraints = query_crdb_for_rows_of_strings( + let referential_constraints = crdb_select( crdb, REFERENTIAL_CONSTRAINTS.as_slice().into(), "information_schema.referential_constraints", @@ -642,7 +648,7 @@ impl InformationSchema { ) .await; - let views = query_crdb_for_rows_of_strings( + let views = crdb_select( crdb, VIEWS.as_slice().into(), "information_schema.views", @@ -650,7 +656,7 @@ impl InformationSchema { ) .await; - let statistics = query_crdb_for_rows_of_strings( + let statistics = crdb_select( crdb, STATISTICS.as_slice().into(), "information_schema.statistics", @@ -658,7 +664,7 @@ impl InformationSchema { ) .await; - let sequences = query_crdb_for_rows_of_strings( + let sequences = crdb_select( crdb, SEQUENCES.as_slice().into(), "information_schema.sequences", @@ -666,7 +672,7 @@ impl InformationSchema { ) .await; - let pg_indexes = query_crdb_for_rows_of_strings( + let pg_indexes = crdb_select( crdb, PG_INDEXES.as_slice().into(), "pg_indexes", @@ -674,7 +680,7 @@ impl InformationSchema { ) .await; - let tables = query_crdb_for_rows_of_strings( + let tables = crdb_select( crdb, TABLES.as_slice().into(), "information_schema.tables", @@ -682,17 +688,11 @@ impl InformationSchema { ) .await; - let table_constraints = query_crdb_for_rows_of_strings( - crdb, - TABLE_CONSTRAINTS.as_slice().into(), - "information_schema.table_constraints", - Some("table_schema = 'public'"), - ) - .await; + let table_constraints = + Self::show_constraints_all_tables(&tables, crdb).await; Self { columns, - check_constraints, constraint_column_usage, key_column_usage, referential_constraints, @@ -705,6 +705,33 @@ impl InformationSchema { } } + async fn show_constraints_all_tables( + tables: &Vec, + crdb: &CockroachInstance, + ) -> BTreeMap> { + let mut map = BTreeMap::new(); + + for table in tables { + let table = &table.values; + let table_catalog = + table[0].expect("table_catalog").unwrap().as_str(); + let table_schema = + table[1].expect("table_schema").unwrap().as_str(); + let table_name = table[2].expect("table_name").unwrap().as_str(); + let table_type = table[3].expect("table_type").unwrap().as_str(); + + if table_type != "BASE TABLE" { + continue; + } + + let table_name = + format!("{}.{}.{}", table_catalog, table_schema, table_name); + let rows = crdb_show_constraints(crdb, &table_name).await; + map.insert(table_name, rows); + } + map + } + // This would normally be quite an expensive operation, but we expect it'll // at least be slightly cheaper for the freshly populated DB, which // shouldn't have that many records yet. @@ -731,13 +758,9 @@ impl InformationSchema { let table_name = format!("{}.{}.{}", table_catalog, table_schema, table_name); info!(log, "Querying table: {table_name}"); - let rows = query_crdb_for_rows_of_strings( - crdb, - ColumnSelector::Star, - &table_name, - None, - ) - .await; + let rows = + crdb_select(crdb, ColumnSelector::Star, &table_name, None) + .await; info!(log, "Saw data: {rows:?}"); map.insert(table_name, rows); } @@ -1012,6 +1035,47 @@ async fn compare_table_differing_constraint() { ) .await; - assert_ne!(schema1.check_constraints, schema2.check_constraints); + assert_ne!(schema1.table_constraints, schema2.table_constraints); + logctx.cleanup_successful(); +} + +#[tokio::test] +async fn compare_table_differing_not_null_order() { + let config = load_test_config(); + let logctx = LogContext::new( + "compare_table_differing_not_null_order", + &config.pkg.log, + ); + let log = &logctx.log; + + let schema1 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.pet ( id UUID PRIMARY KEY ); + CREATE TABLE omicron.public.employee ( + id UUID PRIMARY KEY, + name TEXT NOT NULL, + hobbies TEXT + ); + ", + ) + .await; + + let schema2 = get_information_schema( + log, + " + CREATE DATABASE omicron; + CREATE TABLE omicron.public.employee ( + id UUID PRIMARY KEY, + name TEXT NOT NULL, + hobbies TEXT + ); + CREATE TABLE omicron.public.pet ( id UUID PRIMARY KEY ); + ", + ) + .await; + + schema1.pretty_assert_eq(&schema2); logctx.cleanup_successful(); } diff --git a/nexus/tests/integration_tests/snapshots.rs b/nexus/tests/integration_tests/snapshots.rs index 68f4cdadd2..1dd32e6769 100644 --- a/nexus/tests/integration_tests/snapshots.rs +++ b/nexus/tests/integration_tests/snapshots.rs @@ -1041,7 +1041,25 @@ async fn test_create_snapshot_record_idempotent( external::Error::ObjectAlreadyExists { .. }, )); - // Test project_delete_snapshot is idempotent + // Move snapshot from Creating to Ready + + let (.., authz_snapshot, db_snapshot) = LookupPath::new(&opctx, &datastore) + .snapshot_id(snapshot_created_1.id()) + .fetch_for(authz::Action::Modify) + .await + .unwrap(); + + datastore + .project_snapshot_update_state( + &opctx, + &authz_snapshot, + db_snapshot.gen, + db::model::SnapshotState::Ready, + ) + .await + .unwrap(); + + // Grab the new snapshot (so generation number is updated) let (.., authz_snapshot, db_snapshot) = LookupPath::new(&opctx, &datastore) .snapshot_id(snapshot_created_1.id()) @@ -1049,6 +1067,8 @@ async fn test_create_snapshot_record_idempotent( .await .unwrap(); + // Test project_delete_snapshot is idempotent for the same input + datastore .project_delete_snapshot( &opctx, @@ -1064,6 +1084,16 @@ async fn test_create_snapshot_record_idempotent( .await .unwrap(); + { + // Ensure the snapshot is gone + let r = LookupPath::new(&opctx, &datastore) + .snapshot_id(snapshot_created_1.id()) + .fetch_for(authz::Action::Read) + .await; + + assert!(r.is_err()); + } + datastore .project_delete_snapshot( &opctx, @@ -1287,45 +1317,47 @@ async fn test_multiple_deletes_not_sent(cptestctx: &ControlPlaneTestContext) { .await .unwrap(); - let resources_1 = match resources_1 { - db::datastore::CrucibleResources::V1(_) => panic!("using old style!"), - db::datastore::CrucibleResources::V2(resources_1) => resources_1, - }; - let resources_2 = match resources_2 { - db::datastore::CrucibleResources::V1(_) => panic!("using old style!"), - db::datastore::CrucibleResources::V2(resources_2) => resources_2, - }; - let resources_3 = match resources_3 { - db::datastore::CrucibleResources::V1(_) => panic!("using old style!"), - db::datastore::CrucibleResources::V2(resources_3) => resources_3, - }; + let resources_1_datasets_and_regions = + datastore.regions_to_delete(&resources_1).await.unwrap(); + let resources_1_datasets_and_snapshots = + datastore.snapshots_to_delete(&resources_1).await.unwrap(); + + let resources_2_datasets_and_regions = + datastore.regions_to_delete(&resources_2).await.unwrap(); + let resources_2_datasets_and_snapshots = + datastore.snapshots_to_delete(&resources_2).await.unwrap(); + + let resources_3_datasets_and_regions = + datastore.regions_to_delete(&resources_3).await.unwrap(); + let resources_3_datasets_and_snapshots = + datastore.snapshots_to_delete(&resources_3).await.unwrap(); // No region deletions yet, these are just snapshot deletes - assert!(resources_1.datasets_and_regions.is_empty()); - assert!(resources_2.datasets_and_regions.is_empty()); - assert!(resources_3.datasets_and_regions.is_empty()); + assert!(resources_1_datasets_and_regions.is_empty()); + assert!(resources_2_datasets_and_regions.is_empty()); + assert!(resources_3_datasets_and_regions.is_empty()); // But there are snapshots to delete - assert!(!resources_1.snapshots_to_delete.is_empty()); - assert!(!resources_2.snapshots_to_delete.is_empty()); - assert!(!resources_3.snapshots_to_delete.is_empty()); + assert!(!resources_1_datasets_and_snapshots.is_empty()); + assert!(!resources_2_datasets_and_snapshots.is_empty()); + assert!(!resources_3_datasets_and_snapshots.is_empty()); // Assert there are no overlaps in the snapshots_to_delete to delete. - for tuple in &resources_1.snapshots_to_delete { - assert!(!resources_2.snapshots_to_delete.contains(tuple)); - assert!(!resources_3.snapshots_to_delete.contains(tuple)); + for tuple in &resources_1_datasets_and_snapshots { + assert!(!resources_2_datasets_and_snapshots.contains(tuple)); + assert!(!resources_3_datasets_and_snapshots.contains(tuple)); } - for tuple in &resources_2.snapshots_to_delete { - assert!(!resources_1.snapshots_to_delete.contains(tuple)); - assert!(!resources_3.snapshots_to_delete.contains(tuple)); + for tuple in &resources_2_datasets_and_snapshots { + assert!(!resources_1_datasets_and_snapshots.contains(tuple)); + assert!(!resources_3_datasets_and_snapshots.contains(tuple)); } - for tuple in &resources_3.snapshots_to_delete { - assert!(!resources_1.snapshots_to_delete.contains(tuple)); - assert!(!resources_2.snapshots_to_delete.contains(tuple)); + for tuple in &resources_3_datasets_and_snapshots { + assert!(!resources_1_datasets_and_snapshots.contains(tuple)); + assert!(!resources_2_datasets_and_snapshots.contains(tuple)); } } diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index 3d3d6c9f5f..fada45694d 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -10,8 +10,10 @@ use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder}; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params::{ Address, AddressConfig, AddressLotBlockCreate, AddressLotCreate, - LinkConfig, LldpServiceConfig, Route, RouteConfig, SwitchInterfaceConfig, - SwitchInterfaceKind, SwitchPortApplySettings, SwitchPortSettingsCreate, + BgpAnnounceSetCreate, BgpAnnouncementCreate, BgpConfigCreate, + BgpPeerConfig, LinkConfig, LinkFec, LinkSpeed, LldpServiceConfig, Route, + RouteConfig, SwitchInterfaceConfig, SwitchInterfaceKind, + SwitchPortApplySettings, SwitchPortSettingsCreate, }; use nexus_types::external_api::views::Rack; use omicron_common::api::external::{ @@ -33,10 +35,16 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { description: "an address parking lot".into(), }, kind: AddressLotKind::Infra, - blocks: vec![AddressLotBlockCreate { - first_address: "203.0.113.10".parse().unwrap(), - last_address: "203.0.113.20".parse().unwrap(), - }], + blocks: vec![ + AddressLotBlockCreate { + first_address: "203.0.113.10".parse().unwrap(), + last_address: "203.0.113.20".parse().unwrap(), + }, + AddressLotBlockCreate { + first_address: "1.2.3.0".parse().unwrap(), + last_address: "1.2.3.255".parse().unwrap(), + }, + ], }; NexusRequest::objects_post( @@ -49,6 +57,49 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .await .unwrap(); + // Create BGP announce set + let announce_set = BgpAnnounceSetCreate { + identity: IdentityMetadataCreateParams { + name: "instances".parse().unwrap(), + description: "autonomous system 47 announcements".into(), + }, + announcement: vec![BgpAnnouncementCreate { + address_lot_block: NameOrId::Name("parkinglot".parse().unwrap()), + network: "1.2.3.0/24".parse().unwrap(), + }], + }; + + NexusRequest::objects_post( + client, + "/v1/system/networking/bgp-announce", + &announce_set, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + + // Create BGP config + let bgp_config = BgpConfigCreate { + identity: IdentityMetadataCreateParams { + name: "as47".parse().unwrap(), + description: "autonomous system 47".into(), + }, + bgp_announce_set_id: NameOrId::Name("instances".parse().unwrap()), + asn: 47, + vrf: None, + }; + + NexusRequest::objects_post( + client, + "/v1/system/networking/bgp", + &bgp_config, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + // Create port settings let mut settings = SwitchPortSettingsCreate::new(IdentityMetadataCreateParams { @@ -61,6 +112,8 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { LinkConfig { mtu: 4700, lldp: LldpServiceConfig { enabled: false, lldp_config: None }, + fec: LinkFec::None, + speed: LinkSpeed::Speed100G, }, ); // interfaces @@ -191,6 +244,33 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .parsed_body() .unwrap(); + // Update port settings. Should not see conflict. + settings.bgp_peers.insert( + "phy0".into(), + BgpPeerConfig { + bgp_config: NameOrId::Name("as47".parse().unwrap()), //TODO + bgp_announce_set: NameOrId::Name("instances".parse().unwrap()), //TODO + interface_name: "phy0".to_string(), + addr: "1.2.3.4".parse().unwrap(), + hold_time: 6, + idle_hold_time: 6, + delay_open: 0, + connect_retry: 3, + keepalive: 2, + }, + ); + let _created: SwitchPortSettingsView = NexusRequest::objects_post( + client, + "/v1/system/networking/switch-port-settings", + &settings, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + // There should be one switch port to begin with, see // Server::start_and_populate in nexus/src/lib.rs diff --git a/nexus/tests/integration_tests/volume_management.rs b/nexus/tests/integration_tests/volume_management.rs index e263593def..24a0e5591b 100644 --- a/nexus/tests/integration_tests/volume_management.rs +++ b/nexus/tests/integration_tests/volume_management.rs @@ -1103,8 +1103,6 @@ async fn test_create_image_from_snapshot_delete( assert!(!disk_test.crucible_resources_deleted().await); // Delete the image - // TODO-unimplemented - /* let image_url = "/v1/images/debian-11"; NexusRequest::object_delete(client, &image_url) .authn_as(AuthnMode::PrivilegedUser) @@ -1114,7 +1112,213 @@ async fn test_create_image_from_snapshot_delete( // Assert everything was cleaned up assert!(disk_test.crucible_resources_deleted().await); - */ +} + +enum DeleteImageTestParam { + Image, + Disk, + Snapshot, +} + +async fn delete_image_test( + cptestctx: &ControlPlaneTestContext, + order: &[DeleteImageTestParam], +) { + // 1. Create a blank disk + // 2. Take a snapshot of that disk + // 3. Create an image from that snapshot + // 4. Delete each of these items in some order + + let disk_test = DiskTest::new(&cptestctx).await; + + let client = &cptestctx.external_client; + populate_ip_pool(&client, "default", None).await; + create_org_and_project(client).await; + + let disks_url = get_disks_url(); + + // Create a blank disk + + let disk_size = ByteCount::from_gibibytes_u32(2); + let base_disk_name: Name = "base-disk".parse().unwrap(); + let base_disk = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: base_disk_name.clone(), + description: String::from("all your base disk are belong to us"), + }, + disk_source: params::DiskSource::Blank { + block_size: params::BlockSize::try_from(512).unwrap(), + }, + size: disk_size, + }; + + let _base_disk: Disk = NexusRequest::new( + RequestBuilder::new(client, Method::POST, &disks_url) + .body(Some(&base_disk)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + + // Issue snapshot request + let snapshots_url = format!("/v1/snapshots?project={}", PROJECT_NAME); + + let snapshot: views::Snapshot = object_create( + client, + &snapshots_url, + ¶ms::SnapshotCreate { + identity: IdentityMetadataCreateParams { + name: "a-snapshot".parse().unwrap(), + description: String::from("you are on the way to destruction"), + }, + disk: base_disk_name.clone().into(), + }, + ) + .await; + + // Create an image from the snapshot + let image_create_params = params::ImageCreate { + identity: IdentityMetadataCreateParams { + name: "debian-11".parse().unwrap(), + description: String::from( + "you have no chance to survive make your time", + ), + }, + source: params::ImageSource::Snapshot { id: snapshot.identity.id }, + os: "debian".parse().unwrap(), + version: "12".into(), + }; + + let _image: views::Image = + NexusRequest::objects_post(client, "/v1/images", &image_create_params) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + + assert_eq!(order.len(), 3); + for item in order { + // Still some crucible resources + assert!(!disk_test.crucible_resources_deleted().await); + + match item { + DeleteImageTestParam::Image => { + let image_url = "/v1/images/debian-11"; + NexusRequest::object_delete(client, &image_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete image"); + } + + DeleteImageTestParam::Disk => { + NexusRequest::object_delete(client, &get_disk_url("base-disk")) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete disk"); + } + + DeleteImageTestParam::Snapshot => { + let snapshot_url = get_snapshot_url("a-snapshot"); + NexusRequest::object_delete(client, &snapshot_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete snapshot"); + } + } + } + + // Assert everything was cleaned up + assert!(disk_test.crucible_resources_deleted().await); +} + +// Make sure that whatever order disks, images, and snapshots are deleted, the +// Crucible resource accounting that Nexus does is correct. + +#[nexus_test] +async fn test_delete_image_order_1(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Disk, + DeleteImageTestParam::Image, + DeleteImageTestParam::Snapshot, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_2(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Disk, + DeleteImageTestParam::Snapshot, + DeleteImageTestParam::Image, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_3(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Image, + DeleteImageTestParam::Disk, + DeleteImageTestParam::Snapshot, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_4(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Image, + DeleteImageTestParam::Snapshot, + DeleteImageTestParam::Disk, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_5(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Snapshot, + DeleteImageTestParam::Disk, + DeleteImageTestParam::Image, + ], + ) + .await; +} + +#[nexus_test] +async fn test_delete_image_order_6(cptestctx: &ControlPlaneTestContext) { + delete_image_test( + cptestctx, + &[ + DeleteImageTestParam::Snapshot, + DeleteImageTestParam::Image, + DeleteImageTestParam::Disk, + ], + ) + .await; } // A test function to create a volume with the provided read only parent. @@ -1814,6 +2018,44 @@ async fn test_volume_checkout_updates_sparse_mid_multiple_gen( volume_match_gen(new_vol, vec![Some(8), None, Some(10)]); } +#[nexus_test] +async fn test_volume_checkout_randomize_ids_only_read_only( + cptestctx: &ControlPlaneTestContext, +) { + // Verify that a volume_checkout_randomize_ids will not work for + // non-read-only Regions + let nexus = &cptestctx.server.apictx().nexus; + let datastore = nexus.datastore(); + let volume_id = Uuid::new_v4(); + let block_size = 512; + + // Create three sub_vols. + let subvol_one = create_region(block_size, 7, Uuid::new_v4()); + let subvol_two = create_region(block_size, 7, Uuid::new_v4()); + let subvol_three = create_region(block_size, 7, Uuid::new_v4()); + + // Make the volume with our three sub_volumes + let volume_construction_request = VolumeConstructionRequest::Volume { + id: volume_id, + block_size, + sub_volumes: vec![subvol_one, subvol_two, subvol_three], + read_only_parent: None, + }; + + // Insert the volume into the database. + datastore + .volume_create(nexus_db_model::Volume::new( + volume_id, + serde_json::to_string(&volume_construction_request).unwrap(), + )) + .await + .unwrap(); + + // volume_checkout_randomize_ids should fail + let r = datastore.volume_checkout_randomize_ids(volume_id).await; + assert!(r.is_err()); +} + /// Test that the Crucible agent's port reuse does not confuse /// `decrease_crucible_resource_count_and_soft_delete_volume`, due to the /// `[ipv6]:port` targets being reused. @@ -1977,17 +2219,12 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { assert_eq!(region_snapshot.deleting, true); } - match cr { - nexus_db_queries::db::datastore::CrucibleResources::V1(cr) => { - assert!(cr.datasets_and_regions.is_empty()); - assert_eq!(cr.datasets_and_snapshots.len(), 3); - } + let datasets_and_regions = datastore.regions_to_delete(&cr).await.unwrap(); + let datasets_and_snapshots = + datastore.snapshots_to_delete(&cr).await.unwrap(); - nexus_db_queries::db::datastore::CrucibleResources::V2(cr) => { - assert!(cr.datasets_and_regions.is_empty()); - assert_eq!(cr.snapshots_to_delete.len(), 3); - } - } + assert!(datasets_and_regions.is_empty()); + assert_eq!(datasets_and_snapshots.len(), 3); // Now, let's say we're at a spot where the running snapshots have been // deleted, but before volume_hard_delete or region_snapshot_remove are @@ -2108,17 +2345,12 @@ async fn test_keep_your_targets_straight(cptestctx: &ControlPlaneTestContext) { assert_eq!(region_snapshot.deleting, true); } - match cr { - nexus_db_queries::db::datastore::CrucibleResources::V1(cr) => { - assert!(cr.datasets_and_regions.is_empty()); - assert_eq!(cr.datasets_and_snapshots.len(), 3); - } + let datasets_and_regions = datastore.regions_to_delete(&cr).await.unwrap(); + let datasets_and_snapshots = + datastore.snapshots_to_delete(&cr).await.unwrap(); - nexus_db_queries::db::datastore::CrucibleResources::V2(cr) => { - assert!(cr.datasets_and_regions.is_empty()); - assert_eq!(cr.snapshots_to_delete.len(), 3); - } - } + assert!(datasets_and_regions.is_empty()); + assert_eq!(datasets_and_snapshots.len(), 3); } #[nexus_test] diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 1d7f5556c2..e55eaa4df6 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -145,6 +145,14 @@ networking_address_lot_block_list GET /v1/system/networking/address- networking_address_lot_create POST /v1/system/networking/address-lot networking_address_lot_delete DELETE /v1/system/networking/address-lot/{address_lot} networking_address_lot_list GET /v1/system/networking/address-lot +networking_bgp_announce_set_create POST /v1/system/networking/bgp-announce +networking_bgp_announce_set_delete DELETE /v1/system/networking/bgp-announce +networking_bgp_announce_set_list GET /v1/system/networking/bgp-announce +networking_bgp_config_create POST /v1/system/networking/bgp +networking_bgp_config_delete DELETE /v1/system/networking/bgp +networking_bgp_config_list GET /v1/system/networking/bgp +networking_bgp_imported_routes_ipv4 GET /v1/system/networking/bgp-routes-ipv4 +networking_bgp_status GET /v1/system/networking/bgp-status networking_loopback_address_create POST /v1/system/networking/loopback-address networking_loopback_address_delete DELETE /v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask} networking_loopback_address_list GET /v1/system/networking/loopback-address diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index b4e0e705d8..a0169ae777 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -1325,6 +1325,42 @@ pub enum SwitchPortGeometry { Sfp28x4, } +/// The forward error correction mode of a link. +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LinkFec { + /// Firecode foward error correction. + Firecode, + /// No forward error correction. + None, + /// Reed-Solomon forward error correction. + Rs, +} + +/// The speed of a link. +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LinkSpeed { + /// Zero gigabits per second. + Speed0G, + /// 1 gigabit per second. + Speed1G, + /// 10 gigabits per second. + Speed10G, + /// 25 gigabits per second. + Speed25G, + /// 40 gigabits per second. + Speed40G, + /// 50 gigabits per second. + Speed50G, + /// 100 gigabits per second. + Speed100G, + /// 200 gigabits per second. + Speed200G, + /// 400 gigabits per second. + Speed400G, +} + /// Switch link configuration. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct LinkConfig { @@ -1333,6 +1369,12 @@ pub struct LinkConfig { /// The link-layer discovery protocol (LLDP) configuration for the link. pub lldp: LldpServiceConfig, + + /// The forward error correction mode of the link. + pub fec: LinkFec, + + /// The speed of the link. + pub speed: LinkSpeed, } /// The LLDP configuration associated with a port. LLDP may be either enabled or @@ -1406,6 +1448,20 @@ pub struct Route { pub vid: Option, } +/// Select a BGP config by a name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpConfigSelector { + /// A name or id to use when selecting BGP config. + pub name_or_id: NameOrId, +} + +/// List BGP configs with an optional name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpConfigListSelector { + /// A name or id to use when selecting BGP config. + pub name_or_id: Option, +} + /// A BGP peer configuration for an interface. Includes the set of announcements /// that will be advertised to the peer identified by `addr`. The `bgp_config` /// parameter is a reference to global BGP parameters. The `interface_name` @@ -1427,21 +1483,59 @@ pub struct BgpPeerConfig { /// The address of the host to peer with. pub addr: IpAddr, + + /// How long to hold peer connections between keppalives (seconds). + pub hold_time: u32, + + /// How long to hold a peer in idle before attempting a new session + /// (seconds). + pub idle_hold_time: u32, + + /// How long to delay sending an open request after establishing a TCP + /// session (seconds). + pub delay_open: u32, + + /// How long to to wait between TCP connection retries (seconds). + pub connect_retry: u32, + + /// How often to send keepalive requests (seconds). + pub keepalive: u32, } /// Parameters for creating a named set of BGP announcements. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct CreateBgpAnnounceSet { +pub struct BgpAnnounceSetCreate { #[serde(flatten)] pub identity: IdentityMetadataCreateParams, /// The announcements in this set. - pub announcement: Vec, + pub announcement: Vec, +} + +/// Select a BGP announce set by a name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpAnnounceSetSelector { + /// A name or id to use when selecting BGP port settings + pub name_or_id: NameOrId, +} + +/// List BGP announce set with an optional name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpAnnounceListSelector { + /// A name or id to use when selecting BGP config. + pub name_or_id: Option, +} + +/// Selector used for querying imported BGP routes. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpRouteSelector { + /// The ASN to filter on. Required. + pub asn: u32, } /// A BGP announcement tied to a particular address lot block. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct BgpAnnouncement { +pub struct BgpAnnouncementCreate { /// Address lot this announcement is drawn from. pub address_lot_block: NameOrId, @@ -1452,18 +1546,27 @@ pub struct BgpAnnouncement { /// Parameters for creating a BGP configuration. This includes and autonomous /// system number (ASN) and a virtual routing and forwarding (VRF) identifier. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct CreateBgpConfig { +pub struct BgpConfigCreate { #[serde(flatten)] pub identity: IdentityMetadataCreateParams, /// The autonomous system number of this BGP configuration. pub asn: u32, + pub bgp_announce_set_id: NameOrId, + /// Optional virtual routing and forwarding identifier for this BGP /// configuration. pub vrf: Option, } +/// Select a BGP status information by BGP config id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpStatusSelector { + /// A name or id of the BGP configuration to get status for + pub name_or_id: NameOrId, +} + /// A set of addresses associated with a port configuration. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct AddressConfig { diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index e2a5e3d094..c0991ebb17 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -182,6 +182,7 @@ pub enum ServiceKind { Tfport, BoundaryNtp { snat: SourceNatConfig, nic: ServiceNic }, InternalNtp, + Mgd, } impl fmt::Display for ServiceKind { @@ -200,6 +201,7 @@ impl fmt::Display for ServiceKind { Tfport => "tfport", CruciblePantry => "crucible_pantry", BoundaryNtp { .. } | InternalNtp => "ntp", + Mgd => "mgd", }; write!(f, "{}", s) } diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 682512cc24..6dcf756737 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -241,6 +241,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BootstrapAddressDiscovery": { "oneOf": [ { @@ -333,6 +380,26 @@ "request_id" ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -375,6 +442,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -406,6 +477,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -492,7 +626,7 @@ "description": "Initial rack network configuration", "allOf": [ { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] }, @@ -529,10 +663,17 @@ "recovery_silo" ] }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -543,18 +684,23 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ] }, "RackOperationStatus": { @@ -747,6 +893,28 @@ "user_password_hash" ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -770,67 +938,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "UserId": { "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", "type": "string" diff --git a/openapi/gateway.json b/openapi/gateway.json index 6a8c72c73f..97cb7994aa 100644 --- a/openapi/gateway.json +++ b/openapi/gateway.json @@ -551,6 +551,70 @@ } } }, + "/sp/{type}/{slot}/component/{component}/cfpa": { + "get": { + "summary": "Read the requested CFPA slot from a root of trust.", + "description": "This endpoint is only valid for the `rot` component.", + "operationId": "sp_rot_cfpa_get", + "parameters": [ + { + "in": "path", + "name": "component", + "description": "ID for the component of the SP; this is the internal identifier used by the SP itself to identify its components.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "slot", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "in": "path", + "name": "type", + "required": true, + "schema": { + "$ref": "#/components/schemas/SpType" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetCfpaParams" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RotCfpa" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/sp/{type}/{slot}/component/{component}/clear-status": { "post": { "summary": "Clear status of a component", @@ -598,6 +662,60 @@ } } }, + "/sp/{type}/{slot}/component/{component}/cmpa": { + "get": { + "summary": "Read the CMPA from a root of trust.", + "description": "This endpoint is only valid for the `rot` component.", + "operationId": "sp_rot_cmpa_get", + "parameters": [ + { + "in": "path", + "name": "component", + "description": "ID for the component of the SP; this is the internal identifier used by the SP itself to identify its components.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "slot", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + { + "in": "path", + "name": "type", + "required": true, + "schema": { + "$ref": "#/components/schemas/SpType" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RotCmpa" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/sp/{type}/{slot}/component/{component}/reset": { "post": { "summary": "Reset an SP component (possibly the SP itself).", @@ -1326,6 +1444,17 @@ "request_id" ] }, + "GetCfpaParams": { + "type": "object", + "properties": { + "slot": { + "$ref": "#/components/schemas/RotCfpaSlot" + } + }, + "required": [ + "slot" + ] + }, "HostPhase2Progress": { "oneOf": [ { @@ -2071,6 +2200,78 @@ "A2" ] }, + "RotCfpa": { + "type": "object", + "properties": { + "base64_data": { + "type": "string" + }, + "slot": { + "$ref": "#/components/schemas/RotCfpaSlot" + } + }, + "required": [ + "base64_data", + "slot" + ] + }, + "RotCfpaSlot": { + "oneOf": [ + { + "type": "object", + "properties": { + "slot": { + "type": "string", + "enum": [ + "active" + ] + } + }, + "required": [ + "slot" + ] + }, + { + "type": "object", + "properties": { + "slot": { + "type": "string", + "enum": [ + "inactive" + ] + } + }, + "required": [ + "slot" + ] + }, + { + "type": "object", + "properties": { + "slot": { + "type": "string", + "enum": [ + "scratch" + ] + } + }, + "required": [ + "slot" + ] + } + ] + }, + "RotCmpa": { + "type": "object", + "properties": { + "base64_data": { + "type": "string" + } + }, + "required": [ + "base64_data" + ] + }, "RotSlot": { "oneOf": [ { diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 67db222155..411c52ddff 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -767,6 +767,53 @@ "serial_number" ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BinRangedouble": { "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", "oneOf": [ @@ -3653,6 +3700,26 @@ } ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -3695,6 +3762,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -4038,6 +4109,69 @@ "PhysicalDiskPutResponse": { "type": "object" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -4268,7 +4402,7 @@ "description": "Initial rack network configuration", "allOf": [ { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] }, @@ -4299,10 +4433,17 @@ "services" ] }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -4313,18 +4454,23 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ] }, "RecoverySiloConfig": { @@ -4346,6 +4492,28 @@ "user_password_hash" ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "Saga": { "description": "Sagas\n\nThese are currently only intended for observability by developers. We will eventually want to flesh this out into something more observable for end users.", "type": "object", @@ -4822,6 +4990,20 @@ "required": [ "type" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "mgd" + ] + } + }, + "required": [ + "type" + ] } ] }, @@ -5090,67 +5272,6 @@ "SwitchPutResponse": { "type": "object" }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "UserId": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", diff --git a/openapi/nexus.json b/openapi/nexus.json index 9dda94f283..f1bfa4351f 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -5165,12 +5165,324 @@ } } }, + "/v1/system/networking/bgp": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "List BGP configurations", + "operationId": "networking_bgp_config_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP config.", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpConfigResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + }, + "post": { + "tags": [ + "system/networking" + ], + "summary": "Create a new BGP configuration", + "operationId": "networking_bgp_config_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpConfigCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "system/networking" + ], + "summary": "Delete a BGP configuration", + "operationId": "networking_bgp_config_delete", + "parameters": [ + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP config.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/networking/bgp-announce": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get originated routes for a BGP configuration", + "operationId": "networking_bgp_announce_set_list", + "parameters": [ + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP port settings", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_BgpAnnouncement", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpAnnouncement" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "system/networking" + ], + "summary": "Create a new BGP announce set", + "operationId": "networking_bgp_announce_set_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpAnnounceSetCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpAnnounceSet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "system/networking" + ], + "summary": "Delete a BGP announce set", + "operationId": "networking_bgp_announce_set_delete", + "parameters": [ + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP port settings", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/networking/bgp-routes-ipv4": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get imported IPv4 BGP routes", + "operationId": "networking_bgp_imported_routes_ipv4", + "parameters": [ + { + "in": "query", + "name": "asn", + "description": "The ASN to filter on. Required.", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_BgpImportedRouteIpv4", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpImportedRouteIpv4" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/networking/bgp-status": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get BGP peer status", + "operationId": "networking_bgp_status", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_BgpPeerStatus", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerStatus" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/system/networking/loopback-address": { "get": { "tags": [ "system/networking" ], - "summary": "Get loopback addresses, optionally filtering by id", + "summary": "List loopback addresses", "operationId": "networking_loopback_address_list", "parameters": [ { @@ -7741,40 +8053,289 @@ "$ref": "#/components/schemas/AddressLotBlock" } }, - "lot": { - "description": "The address lot that was created.", + "lot": { + "description": "The address lot that was created.", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLot" + } + ] + } + }, + "required": [ + "blocks", + "lot" + ] + }, + "AddressLotKind": { + "description": "The kind associated with an address lot.", + "oneOf": [ + { + "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", + "type": "string", + "enum": [ + "infra" + ] + }, + { + "description": "Pool address lots are used by IP pools.", + "type": "string", + "enum": [ + "pool" + ] + } + ] + }, + "AddressLotResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLot" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "Baseboard": { + "description": "Properties that uniquely identify an Oxide hardware component", + "type": "object", + "properties": { + "part": { + "type": "string" + }, + "revision": { + "type": "integer", + "format": "int64" + }, + "serial": { + "type": "string" + } + }, + "required": [ + "part", + "revision", + "serial" + ] + }, + "BgpAnnounceSet": { + "description": "Represents a BGP announce set by id. The id can be used with other API calls to view and manage the announce set.", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "id", + "name", + "time_created", + "time_modified" + ] + }, + "BgpAnnounceSetCreate": { + "description": "Parameters for creating a named set of BGP announcements.", + "type": "object", + "properties": { + "announcement": { + "description": "The announcements in this set.", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpAnnouncementCreate" + } + }, + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "announcement", + "description", + "name" + ] + }, + "BgpAnnouncement": { + "description": "A BGP announcement tied to an address lot block.", + "type": "object", + "properties": { + "address_lot_block_id": { + "description": "The address block the IP network being announced is drawn from.", + "type": "string", + "format": "uuid" + }, + "announce_set_id": { + "description": "The id of the set this announcement is a part of.", + "type": "string", + "format": "uuid" + }, + "network": { + "description": "The IP network being announced.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + } + }, + "required": [ + "address_lot_block_id", + "announce_set_id", + "network" + ] + }, + "BgpAnnouncementCreate": { + "description": "A BGP announcement tied to a particular address lot block.", + "type": "object", + "properties": { + "address_lot_block": { + "description": "Address lot this announcement is drawn from.", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + }, + "network": { + "description": "The network being announced.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + } + }, + "required": [ + "address_lot_block", + "network" + ] + }, + "BgpConfig": { + "description": "A base BGP configuration.", + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number of this BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + }, + "vrf": { + "nullable": true, + "description": "Optional virtual routing and forwarding identifier for this BGP configuration.", + "type": "string" + } + }, + "required": [ + "asn", + "description", + "id", + "name", + "time_created", + "time_modified" + ] + }, + "BgpConfigCreate": { + "description": "Parameters for creating a BGP configuration. This includes and autonomous system number (ASN) and a virtual routing and forwarding (VRF) identifier.", + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number of this BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "bgp_announce_set_id": { + "$ref": "#/components/schemas/NameOrId" + }, + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "vrf": { + "nullable": true, + "description": "Optional virtual routing and forwarding identifier for this BGP configuration.", "allOf": [ { - "$ref": "#/components/schemas/AddressLot" + "$ref": "#/components/schemas/Name" } ] } }, "required": [ - "blocks", - "lot" - ] - }, - "AddressLotKind": { - "description": "The kind associated with an address lot.", - "oneOf": [ - { - "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", - "type": "string", - "enum": [ - "infra" - ] - }, - { - "description": "Pool address lots are used by IP pools.", - "type": "string", - "enum": [ - "pool" - ] - } + "asn", + "bgp_announce_set_id", + "description", + "name" ] }, - "AddressLotResultsPage": { + "BgpConfigResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -7782,7 +8343,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/AddressLot" + "$ref": "#/components/schemas/BgpConfig" } }, "next_page": { @@ -7795,25 +8356,43 @@ "items" ] }, - "Baseboard": { - "description": "Properties that uniquely identify an Oxide hardware component", + "BgpImportedRouteIpv4": { + "description": "A route imported from a BGP peer.", "type": "object", "properties": { - "part": { - "type": "string" - }, - "revision": { + "id": { + "description": "BGP identifier of the originating router.", "type": "integer", - "format": "int64" + "format": "uint32", + "minimum": 0 }, - "serial": { - "type": "string" + "nexthop": { + "description": "The nexthop the prefix is reachable through.", + "type": "string", + "format": "ipv4" + }, + "prefix": { + "description": "The destination network prefix.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Net" + } + ] + }, + "switch": { + "description": "Switch the route is imported into.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] } }, "required": [ - "part", - "revision", - "serial" + "id", + "nexthop", + "prefix", + "switch" ] }, "BgpPeerConfig": { @@ -7841,16 +8420,158 @@ } ] }, + "connect_retry": { + "description": "How long to to wait between TCP connection retries (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "delay_open": { + "description": "How long to delay sending an open request after establishing a TCP session (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "hold_time": { + "description": "How long to hold peer connections between keppalives (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "idle_hold_time": { + "description": "How long to hold a peer in idle before attempting a new session (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "interface_name": { "description": "The name of interface to peer on. This is relative to the port configuration this BGP peer configuration is a part of. For example this value could be phy0 to refer to a primary physical interface. Or it could be vlan47 to refer to a VLAN interface.", "type": "string" + }, + "keepalive": { + "description": "How often to send keepalive requests (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 } }, "required": [ "addr", "bgp_announce_set", "bgp_config", - "interface_name" + "connect_retry", + "delay_open", + "hold_time", + "idle_hold_time", + "interface_name", + "keepalive" + ] + }, + "BgpPeerState": { + "description": "The current state of a BGP peer.", + "oneOf": [ + { + "description": "Initial state. Refuse all incomming BGP connections. No resources allocated to peer.", + "type": "string", + "enum": [ + "idle" + ] + }, + { + "description": "Waiting for the TCP connection to be completed.", + "type": "string", + "enum": [ + "connect" + ] + }, + { + "description": "Trying to acquire peer by listening for and accepting a TCP connection.", + "type": "string", + "enum": [ + "active" + ] + }, + { + "description": "Waiting for open message from peer.", + "type": "string", + "enum": [ + "open_sent" + ] + }, + { + "description": "Waiting for keepaliave or notification from peer.", + "type": "string", + "enum": [ + "open_confirm" + ] + }, + { + "description": "Synchronizing with peer.", + "type": "string", + "enum": [ + "session_setup" + ] + }, + { + "description": "Session established. Able to exchange update, notification and keepliave messages with peers.", + "type": "string", + "enum": [ + "established" + ] + } + ] + }, + "BgpPeerStatus": { + "description": "The current status of a BGP peer.", + "type": "object", + "properties": { + "addr": { + "description": "IP address of the peer.", + "type": "string", + "format": "ip" + }, + "local_asn": { + "description": "Local autonomous system number.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "remote_asn": { + "description": "Remote autonomous system number.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "state": { + "description": "State of the peer.", + "allOf": [ + { + "$ref": "#/components/schemas/BgpPeerState" + } + ] + }, + "state_duration_millis": { + "description": "Time of last state change.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "switch": { + "description": "Switch with the peer session.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + } + }, + "required": [ + "addr", + "local_asn", + "remote_asn", + "state", + "state_duration_millis", + "switch" ] }, "BinRangedouble": { @@ -11747,6 +12468,14 @@ "description": "Switch link configuration.", "type": "object", "properties": { + "fec": { + "description": "The forward error correction mode of the link.", + "allOf": [ + { + "$ref": "#/components/schemas/LinkFec" + } + ] + }, "lldp": { "description": "The link-layer discovery protocol (LLDP) configuration for the link.", "allOf": [ @@ -11760,11 +12489,115 @@ "type": "integer", "format": "uint16", "minimum": 0 + }, + "speed": { + "description": "The speed of the link.", + "allOf": [ + { + "$ref": "#/components/schemas/LinkSpeed" + } + ] } }, "required": [ + "fec", "lldp", - "mtu" + "mtu", + "speed" + ] + }, + "LinkFec": { + "description": "The forward error correction mode of a link.", + "oneOf": [ + { + "description": "Firecode foward error correction.", + "type": "string", + "enum": [ + "firecode" + ] + }, + { + "description": "No forward error correction.", + "type": "string", + "enum": [ + "none" + ] + }, + { + "description": "Reed-Solomon forward error correction.", + "type": "string", + "enum": [ + "rs" + ] + } + ] + }, + "LinkSpeed": { + "description": "The speed of a link.", + "oneOf": [ + { + "description": "Zero gigabits per second.", + "type": "string", + "enum": [ + "speed0_g" + ] + }, + { + "description": "1 gigabit per second.", + "type": "string", + "enum": [ + "speed1_g" + ] + }, + { + "description": "10 gigabits per second.", + "type": "string", + "enum": [ + "speed10_g" + ] + }, + { + "description": "25 gigabits per second.", + "type": "string", + "enum": [ + "speed25_g" + ] + }, + { + "description": "40 gigabits per second.", + "type": "string", + "enum": [ + "speed40_g" + ] + }, + { + "description": "50 gigabits per second.", + "type": "string", + "enum": [ + "speed50_g" + ] + }, + { + "description": "100 gigabits per second.", + "type": "string", + "enum": [ + "speed100_g" + ] + }, + { + "description": "200 gigabits per second.", + "type": "string", + "enum": [ + "speed200_g" + ] + }, + { + "description": "400 gigabits per second.", + "type": "string", + "enum": [ + "speed400_g" + ] + } ] }, "LldpServiceConfig": { @@ -13539,6 +14372,25 @@ } ] }, + "SwitchLocation": { + "description": "Identifies switch physical location", + "oneOf": [ + { + "description": "Switch in upper slot", + "type": "string", + "enum": [ + "switch0" + ] + }, + { + "description": "Switch in lower slot", + "type": "string", + "enum": [ + "switch1" + ] + } + ] + }, "SwitchPort": { "description": "A switch port represents a physical external port on a rack switch.", "type": "object", @@ -13635,11 +14487,6 @@ "type": "string", "format": "ip" }, - "bgp_announce_set_id": { - "description": "The id for the set of prefixes announced in this peer configuration.", - "type": "string", - "format": "uuid" - }, "bgp_config_id": { "description": "The id of the global BGP configuration referenced by this peer configuration.", "type": "string", @@ -13657,7 +14504,6 @@ }, "required": [ "addr", - "bgp_announce_set_id", "bgp_config_id", "interface_name", "port_settings_id" diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 56437ab283..486662853c 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -289,6 +289,55 @@ } } }, + "/network-bootstore-config": { + "get": { + "summary": "This API endpoint is only reading the local sled agent's view of the", + "description": "bootstore. The boostore is a distributed data store that is eventually consistent. Reads from individual nodes may not represent the latest state.", + "operationId": "read_network_bootstore_config_cache", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "write_network_bootstore_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/services": { "put": { "operationId": "services_put", @@ -338,6 +387,32 @@ } } }, + "/switch-ports": { + "post": { + "operationId": "uplink_ensure", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SwitchPorts" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/timesync": { "get": { "operationId": "timesync_get", @@ -863,6 +938,53 @@ } }, "schemas": { + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BundleUtilization": { "description": "The portion of a debug dataset used for zone bundles.", "type": "object", @@ -1194,6 +1316,36 @@ "vni" ] }, + "DhcpConfig": { + "description": "DHCP configuration for a port\n\nNot present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we use `InstanceRuntimeState::hostname` for this value.", + "type": "object", + "properties": { + "dns_servers": { + "description": "DNS servers to send to the instance\n\n(DHCPv4 option 6; DHCPv6 option 23)", + "type": "array", + "items": { + "type": "string", + "format": "ip" + } + }, + "host_domain": { + "nullable": true, + "description": "DNS zone this instance's hostname belongs to (e.g. the `project.example` part of `instance1.project.example`)\n\n(DHCPv4 option 15; used in DHCPv6 option 39)", + "type": "string" + }, + "search_domains": { + "description": "DNS search domains\n\n(DHCPv4 option 119; DHCPv6 option 24)", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "dns_servers", + "search_domains" + ] + }, "DiskEnsureBody": { "description": "Sent from to a sled agent to establish the runtime state of a Disk", "type": "object", @@ -1571,6 +1723,54 @@ "secs" ] }, + "EarlyNetworkConfig": { + "description": "Network configuration required to bring up the control plane\n\nThe fields in this structure are those from [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This is just for the initial rack configuration and cold boot purposes. Updates come from Nexus.", + "type": "object", + "properties": { + "body": { + "$ref": "#/components/schemas/EarlyNetworkConfigBody" + }, + "generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "body", + "generation", + "schema_version" + ] + }, + "EarlyNetworkConfigBody": { + "description": "This is the actual configuration of EarlyNetworking.\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "type": "object", + "properties": { + "ntp_servers": { + "description": "The external NTP server addresses.", + "type": "array", + "items": { + "type": "string" + } + }, + "rack_network_config": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RackNetworkConfigV1" + } + ] + } + }, + "required": [ + "ntp_servers" + ] + }, "Error": { "description": "Error information from a response.", "type": "object", @@ -1637,6 +1837,26 @@ } ] }, + "HostPortConfig": { + "type": "object", + "properties": { + "addrs": { + "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "port": { + "description": "Switchport to use for external connectivity", + "type": "string" + } + }, + "required": [ + "addrs", + "port" + ] + }, "InstanceCpuCount": { "description": "The number of CPUs in an Instance", "type": "integer", @@ -1697,6 +1917,9 @@ "nullable": true, "type": "string" }, + "dhcp_config": { + "$ref": "#/components/schemas/DhcpConfig" + }, "disks": { "type": "array", "items": { @@ -1731,6 +1954,7 @@ } }, "required": [ + "dhcp_config", "disks", "external_ips", "firewall_rules", @@ -2099,6 +2323,26 @@ } ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "Ipv4Net": { "example": "192.168.1.0/24", "title": "An IPv4 subnet", @@ -2106,6 +2350,10 @@ "type": "string", "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$" }, + "Ipv4Network": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\/(3[0-2]|[0-2]?[0-9])$" + }, "Ipv6Net": { "example": "fd12:3456::/64", "title": "An IPv6 subnet", @@ -2113,6 +2361,10 @@ "type": "string", "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "KnownArtifactKind": { "description": "Kinds of update artifacts, as used by Nexus to determine what updates are available and by sled-agent to determine how to apply an update when asked.", "type": "string", @@ -2247,6 +2499,93 @@ } ] }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, + "PortFec": { + "description": "Switchport FEC options", + "type": "string", + "enum": [ + "firecode", + "none", + "rs" + ] + }, + "PortSpeed": { + "description": "Switchport Speed options", + "type": "string", + "enum": [ + "speed0_g", + "speed1_g", + "speed10_g", + "speed25_g", + "speed40_g", + "speed50_g", + "speed100_g", + "speed200_g", + "speed400_g" + ] + }, "PriorityDimension": { "description": "A dimension along with bundles can be sorted, to determine priority.", "oneOf": [ @@ -2275,6 +2614,68 @@ "minItems": 2, "maxItems": 2 }, + "RackNetworkConfigV1": { + "description": "Initial network configuration", + "type": "object", + "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, + "infra_ip_first": { + "description": "First ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "infra_ip_last": { + "description": "Last ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "ports": { + "description": "Uplinks for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/PortConfigV1" + } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" + } + }, + "required": [ + "bgp", + "infra_ip_first", + "infra_ip_last", + "ports", + "rack_subnet" + ] + }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -2789,6 +3190,40 @@ "format": "uint8", "minimum": 0 }, + "SwitchLocation": { + "description": "Identifies switch physical location", + "oneOf": [ + { + "description": "Switch in upper slot", + "type": "string", + "enum": [ + "switch0" + ] + }, + { + "description": "Switch in lower slot", + "type": "string", + "enum": [ + "switch1" + ] + } + ] + }, + "SwitchPorts": { + "description": "A set of switch uplinks.", + "type": "object", + "properties": { + "uplinks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HostPortConfig" + } + } + }, + "required": [ + "uplinks" + ] + }, "TimeSync": { "type": "object", "properties": { diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 0e278c9423..a75c965ad8 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -569,6 +569,24 @@ } } }, + "/reload-config": { + "post": { + "summary": "An endpoint instructing wicketd to reload its SMF config properties.", + "description": "The only expected client of this endpoint is `curl` from wicketd's SMF `refresh` method, but other clients hitting it is harmless.", + "operationId": "post_reload_config", + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/repository": { "put": { "summary": "Upload a TUF repository to the server.", @@ -820,6 +838,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BootstrapSledDescription": { "type": "object", "properties": { @@ -1007,7 +1072,7 @@ "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] } @@ -1321,6 +1386,26 @@ "installable" ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -1363,6 +1448,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -1386,6 +1475,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -1955,7 +2107,7 @@ } }, "rack_network_config": { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } }, "required": [ @@ -1972,10 +2124,17 @@ "type": "string", "format": "uuid" }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -1986,18 +2145,23 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ] }, "RackOperationStatus": { @@ -2314,6 +2478,28 @@ } ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -4439,67 +4625,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "IgnitionCommand": { "description": "Ignition command.", "type": "string", diff --git a/oximeter/db/src/client.rs b/oximeter/db/src/client.rs index ffa5d97d52..92b9ed96bd 100644 --- a/oximeter/db/src/client.rs +++ b/oximeter/db/src/client.rs @@ -35,7 +35,7 @@ use std::collections::BTreeSet; use std::convert::TryFrom; use std::net::SocketAddr; use std::num::NonZeroU32; -use std::sync::Mutex; +use tokio::sync::Mutex; use uuid::Uuid; #[usdt::provider(provider = "clickhouse__client")] @@ -208,16 +208,12 @@ impl Client { &self, name: &TimeseriesName, ) -> Result, Error> { - { - let map = self.schema.lock().unwrap(); - if let Some(s) = map.get(name) { - return Ok(Some(s.clone())); - } + let mut schema = self.schema.lock().await; + if let Some(s) = schema.get(name) { + return Ok(Some(s.clone())); } - // `get_schema` acquires the lock internally, so the above scope is required to avoid - // deadlock. - self.get_schema().await?; - Ok(self.schema.lock().unwrap().get(name).map(Clone::clone)) + self.get_schema_locked(&mut schema).await?; + Ok(schema.get(name).map(Clone::clone)) } /// List timeseries schema, paginated. @@ -374,40 +370,64 @@ impl Client { Ok(res.contains("oximeter_cluster")) } - // Verifies that the schema for a sample matches the schema in the database. + // Verifies that the schema for a sample matches the schema in the database, + // or cache a new one internally. // // If the schema exists in the database, and the sample matches that schema, `None` is // returned. If the schema does not match, an Err is returned (the caller skips the sample in - // this case). If the schema does not _exist_ in the database, Some(schema) is returned, so - // that the caller can insert it into the database at the appropriate time. - async fn verify_sample_schema( + // this case). If the schema does not _exist_ in the database, + // Some((timeseries_name, schema)) is returned, so that the caller can + // insert it into the database at the appropriate time. Note that the schema + // is added to the internal cache, but not inserted into the DB at this + // time. + async fn verify_or_cache_sample_schema( &self, sample: &Sample, - ) -> Result, Error> { - let schema = model::schema_for(sample); - let name = schema.timeseries_name.clone(); - let maybe_new_schema = match self.schema.lock().unwrap().entry(name) { - Entry::Vacant(entry) => Ok(Some(entry.insert(schema).clone())), + ) -> Result, Error> { + let sample_schema = model::schema_for(sample); + let name = sample_schema.timeseries_name.clone(); + let mut schema = self.schema.lock().await; + + // We've taken the lock before we do any checks for schema. First, we + // check if we've already got one in the cache. If not, we update all + // the schema from the database, and then check the map again. If we + // find a schema (which now either came from the cache or the latest + // read of the DB), then we check that the derived schema matches. If + // not, we can insert it in the cache and the DB. + if !schema.contains_key(&name) { + self.get_schema_locked(&mut schema).await?; + } + match schema.entry(name) { Entry::Occupied(entry) => { let existing_schema = entry.get(); - if existing_schema == &schema { + if existing_schema == &sample_schema { Ok(None) } else { - let err = - error_for_schema_mismatch(&schema, &existing_schema); error!( self.log, - "timeseries schema mismatch, sample will be skipped: {}", - err + "timeseries schema mismatch, sample will be skipped"; + "expected" => ?existing_schema, + "actual" => ?sample_schema, + "sample" => ?sample, ); - Err(err) + Err(Error::SchemaMismatch { + expected: existing_schema.clone(), + actual: sample_schema, + }) } } - }?; - Ok(maybe_new_schema.map(|schema| { - serde_json::to_string(&model::DbTimeseriesSchema::from(schema)) - .expect("Failed to convert schema to DB model") - })) + Entry::Vacant(entry) => { + let name = entry.key().clone(); + entry.insert(sample_schema.clone()); + Ok(Some(( + name, + serde_json::to_string(&model::DbTimeseriesSchema::from( + sample_schema, + )) + .expect("Failed to convert schema to DB model"), + ))) + } + } } // Select the timeseries, including keys and field values, that match the given field-selection @@ -462,7 +482,6 @@ impl Client { Ok(timeseries_by_key.into_values().collect()) } - // Initialize ClickHouse with the database and metric table schema. // Execute a generic SQL statement. // // TODO-robustness This currently does no validation of the statement. @@ -503,10 +522,15 @@ impl Client { response } - async fn get_schema(&self) -> Result<(), Error> { + // Get timeseries schema from the database. + // + // Can only be called after acquiring the lock around `self.schema`. + async fn get_schema_locked( + &self, + schema: &mut BTreeMap, + ) -> Result<(), Error> { debug!(self.log, "retrieving timeseries schema from database"); let sql = { - let schema = self.schema.lock().unwrap(); if schema.is_empty() { format!( "SELECT * FROM {db_name}.timeseries_schema FORMAT JSONEachRow;", @@ -545,57 +569,36 @@ impl Client { ); (schema.timeseries_name.clone(), schema) }); - self.schema.lock().unwrap().extend(new); + schema.extend(new); } Ok(()) } -} - -/// A trait allowing a [`Client`] to write data into the timeseries database. -/// -/// The vanilla [`Client`] object allows users to query the timeseries database, returning -/// timeseries samples corresponding to various filtering criteria. This trait segregates the -/// methods required for _writing_ new data into the database, and is intended only for use by the -/// `oximeter-collector` crate. -#[async_trait] -pub trait DbWrite { - /// Insert the given samples into the database. - async fn insert_samples(&self, samples: &[Sample]) -> Result<(), Error>; - - /// Initialize the replicated telemetry database, creating tables as needed. - async fn init_replicated_db(&self) -> Result<(), Error>; - - /// Initialize a single node telemetry database, creating tables as needed. - async fn init_single_node_db(&self) -> Result<(), Error>; - - /// Wipe the ClickHouse database entirely from a single node set up. - async fn wipe_single_node_db(&self) -> Result<(), Error>; - - /// Wipe the ClickHouse database entirely from a replicated set up. - async fn wipe_replicated_db(&self) -> Result<(), Error>; -} -#[async_trait] -impl DbWrite for Client { - /// Insert the given samples into the database. - async fn insert_samples(&self, samples: &[Sample]) -> Result<(), Error> { - debug!(self.log, "unrolling {} total samples", samples.len()); + // Unroll each sample into its consituent rows, after verifying the schema. + // + // Note that this also inserts the schema into the internal cache, if it + // does not already exist there. + async fn unroll_samples(&self, samples: &[Sample]) -> UnrolledSampleRows { let mut seen_timeseries = BTreeSet::new(); let mut rows = BTreeMap::new(); - let mut new_schema = Vec::new(); + let mut new_schema = BTreeMap::new(); for sample in samples.iter() { - match self.verify_sample_schema(sample).await { + match self.verify_or_cache_sample_schema(sample).await { Err(_) => { // Skip the sample, but otherwise do nothing. The error is logged in the above // call. continue; } - Ok(schema) => { - if let Some(schema) = schema { - debug!(self.log, "new timeseries schema: {:?}", schema); - new_schema.push(schema); - } + Ok(None) => {} + Ok(Some((name, schema))) => { + debug!( + self.log, + "new timeseries schema"; + "timeseries_name" => %name, + "schema" => %schema + ); + new_schema.insert(name, schema); } } @@ -623,34 +626,78 @@ impl DbWrite for Client { seen_timeseries.insert(key); } - // Insert the new schema into the database - // - // TODO-robustness There's still a race possible here. If two distinct clients receive new - // but conflicting schema, they will both try to insert those at some point into the schema - // tables. It's not clear how to handle this, since ClickHouse provides no transactions. - // This is unlikely to happen at this point, because the design is such that there will be - // a single `oximeter` instance, which has one client object, connected to a single - // ClickHouse server. But once we start replicating data, the window within which the race - // can occur is much larger, since it includes the time it takes ClickHouse to replicate - // data between nodes. - // - // NOTE: This is an issue even in the case where the schema don't conflict. Two clients may - // receive a sample with a new schema, and both would then try to insert that schema. + UnrolledSampleRows { new_schema, rows } + } + + // Save new schema to the database, or remove them from the cache on + // failure. + // + // This attempts to insert the provided schema into the timeseries schema + // table. If that fails, those schema are _also_ removed from the internal + // cache. + // + // TODO-robustness There's still a race possible here. If two distinct clients receive new + // but conflicting schema, they will both try to insert those at some point into the schema + // tables. It's not clear how to handle this, since ClickHouse provides no transactions. + // This is unlikely to happen at this point, because the design is such that there will be + // a single `oximeter` instance, which has one client object, connected to a single + // ClickHouse server. But once we start replicating data, the window within which the race + // can occur is much larger, since it includes the time it takes ClickHouse to replicate + // data between nodes. + // + // NOTE: This is an issue even in the case where the schema don't conflict. Two clients may + // receive a sample with a new schema, and both would then try to insert that schema. + async fn save_new_schema_or_remove( + &self, + new_schema: BTreeMap, + ) -> Result<(), Error> { if !new_schema.is_empty() { debug!( self.log, "inserting {} new timeseries schema", new_schema.len() ); - let body = format!( - "INSERT INTO {db_name}.timeseries_schema FORMAT JSONEachRow\n{row_data}\n", - db_name = crate::DATABASE_NAME, - row_data = new_schema.join("\n"), + const APPROX_ROW_SIZE: usize = 64; + let mut body = String::with_capacity( + APPROX_ROW_SIZE + APPROX_ROW_SIZE * new_schema.len(), ); - self.execute(body).await?; + body.push_str("INSERT INTO "); + body.push_str(crate::DATABASE_NAME); + body.push_str(".timeseries_schema FORMAT JSONEachRow\n"); + for row_data in new_schema.values() { + body.push_str(row_data); + body.push_str("\n"); + } + + // Try to insert the schema. + // + // If this fails, be sure to remove the schema we've added from the + // internal cache. Since we check the internal cache first for + // schema, if we fail here but _don't_ remove the schema, we'll + // never end up inserting the schema, but we will insert samples. + if let Err(e) = self.execute(body).await { + debug!( + self.log, + "failed to insert new schema, removing from cache"; + "error" => ?e, + ); + let mut schema = self.schema.lock().await; + for name in new_schema.keys() { + schema + .remove(name) + .expect("New schema should have been cached"); + } + return Err(e); + } } + Ok(()) + } - // Insert the actual target/metric field rows and measurement rows. + // Insert unrolled sample rows into the corresponding tables. + async fn insert_unrolled_samples( + &self, + rows: BTreeMap>, + ) -> Result<(), Error> { for (table_name, rows) in rows { let body = format!( "INSERT INTO {table_name} FORMAT JSONEachRow\n{row_data}\n", @@ -663,9 +710,9 @@ impl DbWrite for Client { self.execute(body).await?; debug!( self.log, - "inserted {} rows into table {}", - rows.len(), - table_name + "inserted rows into table"; + "n_rows" => rows.len(), + "table_name" => table_name, ); } @@ -673,6 +720,50 @@ impl DbWrite for Client { // many as one per sample. It's not clear how to structure this in a way that's useful. Ok(()) } +} + +#[derive(Debug)] +struct UnrolledSampleRows { + // The timeseries schema rows, keyed by timeseries name. + new_schema: BTreeMap, + // The rows to insert in all the other tables, keyed by the table name. + rows: BTreeMap>, +} + +/// A trait allowing a [`Client`] to write data into the timeseries database. +/// +/// The vanilla [`Client`] object allows users to query the timeseries database, returning +/// timeseries samples corresponding to various filtering criteria. This trait segregates the +/// methods required for _writing_ new data into the database, and is intended only for use by the +/// `oximeter-collector` crate. +#[async_trait] +pub trait DbWrite { + /// Insert the given samples into the database. + async fn insert_samples(&self, samples: &[Sample]) -> Result<(), Error>; + + /// Initialize the replicated telemetry database, creating tables as needed. + async fn init_replicated_db(&self) -> Result<(), Error>; + + /// Initialize a single node telemetry database, creating tables as needed. + async fn init_single_node_db(&self) -> Result<(), Error>; + + /// Wipe the ClickHouse database entirely from a single node set up. + async fn wipe_single_node_db(&self) -> Result<(), Error>; + + /// Wipe the ClickHouse database entirely from a replicated set up. + async fn wipe_replicated_db(&self) -> Result<(), Error>; +} + +#[async_trait] +impl DbWrite for Client { + /// Insert the given samples into the database. + async fn insert_samples(&self, samples: &[Sample]) -> Result<(), Error> { + debug!(self.log, "unrolling {} total samples", samples.len()); + let UnrolledSampleRows { new_schema, rows } = + self.unroll_samples(samples).await; + self.save_new_schema_or_remove(new_schema).await?; + self.insert_unrolled_samples(rows).await + } /// Initialize the replicated telemetry database, creating tables as needed. async fn init_replicated_db(&self) -> Result<(), Error> { @@ -708,7 +799,7 @@ impl DbWrite for Client { /// Wipe the ClickHouse database entirely from a replicated set up. async fn wipe_replicated_db(&self) -> Result<(), Error> { debug!(self.log, "wiping ClickHouse database"); - let sql = include_str!("./db-wipe-single-node.sql").to_string(); + let sql = include_str!("./db-wipe-replicated.sql").to_string(); self.execute(sql).await } } @@ -730,28 +821,6 @@ async fn handle_db_response( } } -// Generate an error describing a schema mismatch -fn error_for_schema_mismatch( - schema: &TimeseriesSchema, - existing_schema: &TimeseriesSchema, -) -> Error { - let expected = existing_schema - .field_schema - .iter() - .map(|field| (field.name.clone(), field.ty)) - .collect(); - let actual = schema - .field_schema - .iter() - .map(|field| (field.name.clone(), field.ty)) - .collect(); - Error::SchemaMismatch { - name: schema.timeseries_name.to_string(), - expected, - actual, - } -} - #[cfg(test)] mod tests { use super::*; @@ -759,7 +828,9 @@ mod tests { use crate::query::field_table_name; use crate::query::measurement_table_name; use chrono::Utc; - use omicron_test_utils::dev::clickhouse::ClickHouseInstance; + use omicron_test_utils::dev::clickhouse::{ + ClickHouseCluster, ClickHouseInstance, + }; use omicron_test_utils::dev::test_setup_log; use oximeter::histogram::Histogram; use oximeter::test_util; @@ -767,16 +838,13 @@ mod tests { use oximeter::FieldValue; use oximeter::Metric; use oximeter::Target; - use slog::o; use std::net::Ipv4Addr; use std::net::Ipv6Addr; use std::time::Duration; use tokio::time::sleep; use uuid::Uuid; - // NOTE: It's important that each test run the ClickHouse server with different ports. - // The tests each require a clean slate. Previously, we ran the tests in a different thread, - // but we now use a different instance of the server to avoid conflicts. + // NOTE: Each test requires a clean slate. Because of this, tests run sequentially. // // This is at least partially because ClickHouse by default provides pretty weak consistency // guarantees. There are options that allow controlling consistency behavior, but we've not yet @@ -785,8 +853,8 @@ mod tests { // TODO-robustness TODO-correctness: Figure out the ClickHouse options we need. #[tokio::test] - async fn test_build_client() { - let logctx = test_setup_log("test_build_client"); + async fn test_single_node() { + let logctx = test_setup_log("test_single_node"); let log = &logctx.log; // Let the OS assign a port and discover it after ClickHouse starts @@ -795,143 +863,183 @@ mod tests { .expect("Failed to start ClickHouse"); let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let client = Client::new(address, &log); - assert!(!client.is_oximeter_cluster().await.unwrap()); + // Test bad database connection + let client = Client::new("127.0.0.1:443".parse().unwrap(), &log); + assert!(matches!( + client.ping().await, + Err(Error::DatabaseUnavailable(_)) + )); - client.wipe_single_node_db().await.unwrap(); - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); - logctx.cleanup_successful(); - } + // Tests that a new client has started and it is not part of a cluster + is_not_oximeter_cluster_test(address).await.unwrap(); - #[tokio::test] - // TODO(https://github.com/oxidecomputer/omicron/issues/4001): This job fails intermittently - // on the ubuntu CI job with "Failed to detect ClickHouse subprocess within timeout" - #[ignore] - async fn test_build_replicated() { - let logctx = test_setup_log("test_build_replicated"); - let log = &logctx.log; + // Tests that data can be inserted via the client + insert_samples_test(address).await.unwrap(); - // Start all Keeper coordinator nodes - let cur_dir = std::env::current_dir().unwrap(); - let keeper_config = - cur_dir.as_path().join("src/configs/keeper_config.xml"); + // Tests for a schema mismatch + schema_mismatch_test(address).await.unwrap(); - // Start Keeper 1 - let k1_port = 9181; - let k1_id = 1; + // Tests for a schema update + schema_updated_test(address).await.unwrap(); - let mut k1 = ClickHouseInstance::new_keeper( - k1_port, - k1_id, - keeper_config.clone(), - ) - .await - .expect("Failed to start ClickHouse keeper 1"); + // Tests for specific timeseries selection + client_select_timeseries_one_test(address).await.unwrap(); - // Start Keeper 2 - let k2_port = 9182; - let k2_id = 2; + // Tests for specific timeseries selection + field_record_count_test(address).await.unwrap(); - let mut k2 = ClickHouseInstance::new_keeper( - k2_port, - k2_id, - keeper_config.clone(), - ) - .await - .expect("Failed to start ClickHouse keeper 2"); + // ClickHouse regression test + unquoted_64bit_integers_test(address).await.unwrap(); - // Start Keeper 3 - let k3_port = 9183; - let k3_id = 3; + // Tests to verify that we can distinguish between metrics by name + differentiate_by_timeseries_name_test(address).await.unwrap(); - let mut k3 = - ClickHouseInstance::new_keeper(k3_port, k3_id, keeper_config) - .await - .expect("Failed to start ClickHouse keeper 3"); - - // Start all replica nodes - let cur_dir = std::env::current_dir().unwrap(); - let replica_config = - cur_dir.as_path().join("src/configs/replica_config.xml"); - - // Start Replica 1 - let r1_port = 8123; - let r1_tcp_port = 9000; - let r1_interserver_port = 9009; - let r1_name = String::from("oximeter_cluster node 1"); - let r1_number = String::from("01"); - let mut db_1 = ClickHouseInstance::new_replicated( - r1_port, - r1_tcp_port, - r1_interserver_port, - r1_name, - r1_number, - replica_config.clone(), - ) - .await - .expect("Failed to start ClickHouse node 1"); - let r1_address = - SocketAddr::new("127.0.0.1".parse().unwrap(), db_1.port()); - - // Start Replica 2 - let r2_port = 8124; - let r2_tcp_port = 9001; - let r2_interserver_port = 9010; - let r2_name = String::from("oximeter_cluster node 2"); - let r2_number = String::from("02"); - let mut db_2 = ClickHouseInstance::new_replicated( - r2_port, - r2_tcp_port, - r2_interserver_port, - r2_name, - r2_number, - replica_config, + // Tests selecting a single timeseries + select_timeseries_with_select_one_test(address).await.unwrap(); + + // Tests selecting two timeseries + select_timeseries_with_select_one_field_with_multiple_values_test( + address, ) .await - .expect("Failed to start ClickHouse node 2"); - let r2_address = - SocketAddr::new("127.0.0.1".parse().unwrap(), db_2.port()); + .unwrap(); - // Create database in node 1 - let client_1 = Client::new(r1_address, &log); - assert!(client_1.is_oximeter_cluster().await.unwrap()); - client_1 - .init_replicated_db() - .await - .expect("Failed to initialize timeseries database"); + // Tests selecting multiple timeseries + select_timeseries_with_select_multiple_fields_with_multiple_values_test(address).await.unwrap(); - // Wait to make sure data has been synchronised. - // TODO(https://github.com/oxidecomputer/omicron/issues/4001): Waiting for 5 secs is a bit sloppy, - // come up with a better way to do this. - sleep(Duration::from_secs(5)).await; + // Tests selecting all timeseries + select_timeseries_with_all_test(address).await.unwrap(); - // Verify database exists in node 2 - let client_2 = Client::new(r2_address, &log); - assert!(client_2.is_oximeter_cluster().await.unwrap()); - let sql = String::from("SHOW DATABASES FORMAT JSONEachRow;"); + // Tests selecting all timeseries with start time + select_timeseries_with_start_time_test(address).await.unwrap(); - let result = client_2.execute_with_body(sql).await.unwrap(); - assert!(result.contains("oximeter")); + // Tests selecting all timeseries with start time + select_timeseries_with_limit_test(address).await.unwrap(); + + // Tests selecting all timeseries with order + select_timeseries_with_order_test(address).await.unwrap(); + + // Tests schema does not change + get_schema_no_new_values_test(address).await.unwrap(); + + // Tests listing timeseries schema + timeseries_schema_list_test(address).await.unwrap(); + + // Tests listing timeseries + list_timeseries_test(address).await.unwrap(); + + // Tests no changes are made when version is not updated + database_version_update_idempotent_test(address).await.unwrap(); + + // Tests that downgrading is impossible + database_version_will_not_downgrade_test(address).await.unwrap(); + + // Tests old data is dropped if version is updated + database_version_wipes_old_version_test(address).await.unwrap(); + + // Tests schema cache is updated when a new sample is inserted + update_schema_cache_on_new_sample_test(address).await.unwrap(); + + // Tests that we can successfully query all extant datum types from the schema table. + select_all_datum_types_test(address).await.unwrap(); + + // Tests that, when cache new schema but _fail_ to insert them, + // we also remove them from the internal cache. + new_schema_removed_when_not_inserted_test(address).await.unwrap(); + + // Tests for fields and measurements + recall_field_value_bool_test(address).await.unwrap(); + + recall_field_value_u8_test(address).await.unwrap(); + + recall_field_value_i8_test(address).await.unwrap(); + + recall_field_value_u16_test(address).await.unwrap(); + + recall_field_value_i16_test(address).await.unwrap(); + + recall_field_value_u32_test(address).await.unwrap(); + + recall_field_value_i32_test(address).await.unwrap(); + + recall_field_value_u64_test(address).await.unwrap(); + + recall_field_value_i64_test(address).await.unwrap(); + + recall_field_value_string_test(address).await.unwrap(); + + recall_field_value_ipv4addr_test(address).await.unwrap(); + + recall_field_value_ipv6addr_test(address).await.unwrap(); + + recall_field_value_uuid_test(address).await.unwrap(); + + recall_measurement_bool_test(address).await.unwrap(); + + recall_measurement_i8_test(address).await.unwrap(); - k1.cleanup().await.expect("Failed to cleanup ClickHouse keeper 1"); - k2.cleanup().await.expect("Failed to cleanup ClickHouse keeper 2"); - k3.cleanup().await.expect("Failed to cleanup ClickHouse keeper 3"); - db_1.cleanup().await.expect("Failed to cleanup ClickHouse server 1"); - db_2.cleanup().await.expect("Failed to cleanup ClickHouse server 2"); + recall_measurement_u8_test(address).await.unwrap(); + recall_measurement_i16_test(address).await.unwrap(); + + recall_measurement_u16_test(address).await.unwrap(); + + recall_measurement_i32_test(address).await.unwrap(); + + recall_measurement_u32_test(address).await.unwrap(); + + recall_measurement_i64_test(address).await.unwrap(); + + recall_measurement_u64_test(address).await.unwrap(); + + recall_measurement_f32_test(address).await.unwrap(); + + recall_measurement_f64_test(address).await.unwrap(); + + recall_measurement_cumulative_i64_test(address).await.unwrap(); + + recall_measurement_cumulative_u64_test(address).await.unwrap(); + + recall_measurement_cumulative_f64_test(address).await.unwrap(); + + recall_measurement_histogram_i8_test(address).await.unwrap(); + + recall_measurement_histogram_u8_test(address).await.unwrap(); + + recall_measurement_histogram_i16_test(address).await.unwrap(); + + recall_measurement_histogram_u16_test(address).await.unwrap(); + + recall_measurement_histogram_i32_test(address).await.unwrap(); + + recall_measurement_histogram_u32_test(address).await.unwrap(); + + recall_measurement_histogram_i64_test(address).await.unwrap(); + + recall_measurement_histogram_u64_test(address).await.unwrap(); + + recall_measurement_histogram_f64_test(address).await.unwrap(); + + db.cleanup().await.expect("Failed to cleanup ClickHouse server"); logctx.cleanup_successful(); } - #[tokio::test] - async fn test_client_insert() { - let logctx = test_setup_log("test_client_insert"); + async fn is_not_oximeter_cluster_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_is_not_oximeter_cluster"); let log = &logctx.log; - // Let the OS assign a port and discover it after ClickHouse starts - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); + let client = Client::new(address, &log); + assert!(!client.is_oximeter_cluster().await.unwrap()); + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) + } + + async fn insert_samples_test(address: SocketAddr) -> Result<(), Error> { + let logctx = test_setup_log("test_insert_samples"); + let log = &logctx.log; let client = Client::new(address, &log); client @@ -945,9 +1053,10 @@ mod tests { } s }; - client.insert_samples(&samples).await.unwrap(); - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + client.insert_samples(&samples).await?; + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } // This is a target with the same name as that in `lib.rs` used for other tests, but with a @@ -970,1619 +1079,2000 @@ mod tests { } } - #[tokio::test] - async fn test_recall_field_value_bool() { - let field = FieldValue::Bool(true); - let as_json = serde_json::Value::from(1_u64); - test_recall_field_value_impl(field, as_json).await; - } + async fn schema_mismatch_test(address: SocketAddr) -> Result<(), Error> { + let logctx = test_setup_log("test_schema_mismatch"); + let log = &logctx.log; - #[tokio::test] - async fn test_recall_field_value_u8() { - let field = FieldValue::U8(1); - let as_json = serde_json::Value::from(1_u8); - test_recall_field_value_impl(field, as_json).await; - } + let client = Client::new(address, &log); + client + .init_single_node_db() + .await + .expect("Failed to initialize timeseries database"); + let sample = test_util::make_sample(); + client.insert_samples(&[sample]).await.unwrap(); - #[tokio::test] - async fn test_recall_field_value_i8() { - let field = FieldValue::I8(1); - let as_json = serde_json::Value::from(1_i8); - test_recall_field_value_impl(field, as_json).await; + let bad_name = name_mismatch::TestTarget { + name: "first_name".into(), + name2: "second_name".into(), + num: 2, + }; + let metric = test_util::TestMetric { + id: uuid::Uuid::new_v4(), + good: true, + datum: 1, + }; + let sample = Sample::new(&bad_name, &metric).unwrap(); + let result = client.verify_or_cache_sample_schema(&sample).await; + assert!(matches!(result, Err(Error::SchemaMismatch { .. }))); + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_recall_field_value_u16() { - let field = FieldValue::U16(1); - let as_json = serde_json::Value::from(1_u16); - test_recall_field_value_impl(field, as_json).await; - } + async fn schema_updated_test(address: SocketAddr) -> Result<(), Error> { + let logctx = test_setup_log("test_schema_updated"); + let log = &logctx.log; - #[tokio::test] - async fn test_recall_field_value_i16() { - let field = FieldValue::I16(1); - let as_json = serde_json::Value::from(1_i16); - test_recall_field_value_impl(field, as_json).await; - } + let client = Client::new(address, &log); + client + .init_single_node_db() + .await + .expect("Failed to initialize timeseries database"); + let sample = test_util::make_sample(); - #[tokio::test] - async fn test_recall_field_value_u32() { - let field = FieldValue::U32(1); - let as_json = serde_json::Value::from(1_u32); - test_recall_field_value_impl(field, as_json).await; - } + // Verify that this sample is considered new, i.e., we return rows to update the timeseries + // schema table. + let result = + client.verify_or_cache_sample_schema(&sample).await.unwrap(); + assert!( + matches!(result, Some(_)), + "When verifying a new sample, the rows to be inserted should be returned" + ); - #[tokio::test] - async fn test_recall_field_value_i32() { - let field = FieldValue::I32(1); - let as_json = serde_json::Value::from(1_i32); - test_recall_field_value_impl(field, as_json).await; - } - - #[tokio::test] - async fn test_recall_field_value_u64() { - let field = FieldValue::U64(1); - let as_json = serde_json::Value::from(1_u64); - test_recall_field_value_impl(field, as_json).await; - } + // Clear the internal caches of seen schema + client.schema.lock().await.clear(); - #[tokio::test] - async fn test_recall_field_value_i64() { - let field = FieldValue::I64(1); - let as_json = serde_json::Value::from(1_i64); - test_recall_field_value_impl(field, as_json).await; - } + // Insert the new sample + client.insert_samples(&[sample.clone()]).await.unwrap(); - #[tokio::test] - async fn test_recall_field_value_string() { - let field = FieldValue::String("foo".into()); - let as_json = serde_json::Value::from("foo"); - test_recall_field_value_impl(field, as_json).await; - } + // The internal map should now contain both the new timeseries schema + let actual_schema = model::schema_for(&sample); + let timeseries_name = + TimeseriesName::try_from(sample.timeseries_name.as_str()).unwrap(); + let expected_schema = client + .schema + .lock() + .await + .get(×eries_name) + .expect( + "After inserting a new sample, its schema should be included", + ) + .clone(); + assert_eq!( + actual_schema, + expected_schema, + "The timeseries schema for a new sample was not correctly inserted into internal cache", + ); - #[tokio::test] - async fn test_recall_field_value_ipv4addr() { - let field = FieldValue::from(Ipv4Addr::LOCALHOST); - let as_json = serde_json::Value::from( - Ipv4Addr::LOCALHOST.to_ipv6_mapped().to_string(), + // This should no longer return a new row to be inserted for the schema of this sample, as + // any schema have been included above. + let result = + client.verify_or_cache_sample_schema(&sample).await.unwrap(); + assert!( + matches!(result, None), + "After inserting new schema, it should no longer be considered new" ); - test_recall_field_value_impl(field, as_json).await; - } - - #[tokio::test] - async fn test_recall_field_value_ipv6addr() { - let field = FieldValue::from(Ipv6Addr::LOCALHOST); - let as_json = serde_json::Value::from(Ipv6Addr::LOCALHOST.to_string()); - test_recall_field_value_impl(field, as_json).await; - } - #[tokio::test] - async fn test_recall_field_value_uuid() { - let id = Uuid::new_v4(); - let field = FieldValue::from(id); - let as_json = serde_json::Value::from(id.to_string()); - test_recall_field_value_impl(field, as_json).await; + // Verify that it's actually in the database! + let sql = String::from( + "SELECT * FROM oximeter.timeseries_schema FORMAT JSONEachRow;", + ); + let result = client.execute_with_body(sql).await.unwrap(); + let schema = result + .lines() + .map(|line| { + TimeseriesSchema::from( + serde_json::from_str::(&line) + .unwrap(), + ) + }) + .collect::>(); + assert_eq!(schema.len(), 1); + assert_eq!(expected_schema, schema[0]); + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) } - async fn test_recall_field_value_impl( - field_value: FieldValue, - as_json: serde_json::Value, - ) { - let logctx = test_setup_log( - format!("test_recall_field_value_{}", field_value.field_type()) - .as_str(), - ); + async fn client_select_timeseries_one_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_client_select_timeseries_one"); let log = &logctx.log; - // Let the OS assign a port and discover it after ClickHouse starts - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - - let client = Client::new(address, log); + let client = Client::new(address, &log); client .init_single_node_db() .await .expect("Failed to initialize timeseries database"); + let samples = test_util::generate_test_samples(2, 2, 2, 2); + client.insert_samples(&samples).await?; - // Insert a record from this field. - const TIMESERIES_NAME: &str = "foo:bar"; - const TIMESERIES_KEY: u64 = 101; - const FIELD_NAME: &str = "baz"; - - let mut inserted_row = serde_json::Map::new(); - inserted_row - .insert("timeseries_name".to_string(), TIMESERIES_NAME.into()); - inserted_row - .insert("timeseries_key".to_string(), TIMESERIES_KEY.into()); - inserted_row.insert("field_name".to_string(), FIELD_NAME.into()); - inserted_row.insert("field_value".to_string(), as_json); - let inserted_row = serde_json::Value::from(inserted_row); - - let row = serde_json::to_string(&inserted_row).unwrap(); - let field_table = field_table_name(field_value.field_type()); - let insert_sql = format!( - "INSERT INTO oximeter.{field_table} FORMAT JSONEachRow {row}" - ); - client.execute(insert_sql).await.expect("Failed to insert field row"); - - // Select it exactly back out. - let select_sql = format!( - "SELECT * FROM oximeter.{} LIMIT 1 FORMAT {};", - field_table_name(field_value.field_type()), - crate::DATABASE_SELECT_FORMAT, - ); - let body = client - .execute_with_body(select_sql) + let sample = samples.first().unwrap(); + let target_fields = sample.target_fields().collect::>(); + let metric_fields = sample.metric_fields().collect::>(); + let criteria = &[ + format!( + "project_id=={}", + target_fields + .iter() + .find(|f| f.name == "project_id") + .unwrap() + .value + ), + format!( + "instance_id=={}", + target_fields + .iter() + .find(|f| f.name == "instance_id") + .unwrap() + .value + ), + format!("cpu_id=={}", metric_fields[0].value), + ]; + let results = client + .select_timeseries_with( + &sample.timeseries_name, + &criteria.iter().map(|x| x.as_str()).collect::>(), + None, + None, + None, + None, + ) .await - .expect("Failed to select field row"); - let actual_row: serde_json::Value = serde_json::from_str(&body) - .expect("Failed to parse field row JSON"); - println!("{actual_row:?}"); - println!("{inserted_row:?}"); + .unwrap(); + assert_eq!(results.len(), 1, "Expected to find a single timeseries"); + let timeseries = &results[0]; assert_eq!( - actual_row, inserted_row, - "Actual and expected field rows do not match" + timeseries.measurements.len(), + 2, + "Expected 2 samples per timeseries" ); - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + + // Compare measurements themselves + let expected_measurements = + samples.iter().map(|sample| &sample.measurement); + let actual_measurements = timeseries.measurements.iter(); + assert!(actual_measurements + .zip(expected_measurements) + .all(|(first, second)| first == second)); + assert_eq!(timeseries.target.name, "virtual_machine"); + // Compare fields, but order might be different. + fn field_cmp<'a>( + needle: &'a crate::Field, + mut haystack: impl Iterator, + ) -> bool { + needle == haystack.find(|f| f.name == needle.name).unwrap() + } + timeseries + .target + .fields + .iter() + .all(|field| field_cmp(field, sample.target_fields())); + assert_eq!(timeseries.metric.name, "cpu_busy"); + timeseries + .metric + .fields + .iter() + .all(|field| field_cmp(field, sample.metric_fields())); + + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_recall_measurement_bool() { - let datum = Datum::Bool(true); - let as_json = serde_json::Value::from(1_u64); - test_recall_measurement_impl::(datum, None, as_json).await; - } + async fn field_record_count_test(address: SocketAddr) -> Result<(), Error> { + let logctx = test_setup_log("test_field_record_count"); + let log = &logctx.log; - #[tokio::test] - async fn test_recall_measurement_i8() { - let datum = Datum::I8(1); - let as_json = serde_json::Value::from(1_i8); - test_recall_measurement_impl::(datum, None, as_json).await; - } + // This test verifies that the number of records in the field tables is as expected. + // + // Because of the schema change, inserting field records per field per unique timeseries, + // we'd like to exercise the logic of ClickHouse's replacing merge tree engine. + let client = Client::new(address, &log); + client + .init_single_node_db() + .await + .expect("Failed to initialize timeseries database"); + let samples = test_util::generate_test_samples(2, 2, 2, 2); + client.insert_samples(&samples).await?; - #[tokio::test] - async fn test_recall_measurement_u8() { - let datum = Datum::U8(1); - let as_json = serde_json::Value::from(1_u8); - test_recall_measurement_impl::(datum, None, as_json).await; - } + async fn assert_table_count( + client: &Client, + table: &str, + expected_count: usize, + ) { + let body = client + .execute_with_body(format!( + "SELECT COUNT() FROM oximeter.{};", + table + )) + .await + .unwrap(); + let actual_count = + body.lines().next().unwrap().trim().parse::().expect( + "Expected a count of the number of rows from ClickHouse", + ); + assert_eq!(actual_count, expected_count); + } - #[tokio::test] - async fn test_recall_measurement_i16() { - let datum = Datum::I16(1); - let as_json = serde_json::Value::from(1_i16); - test_recall_measurement_impl::(datum, None, as_json).await; - } + // There should be (2 projects * 2 instances * 2 cpus) == 8 timeseries. For each of these + // timeseries, there are 2 UUID fields, `project_id` and `instance_id`. So 16 UUID records. + assert_table_count(&client, "fields_uuid", 16).await; - #[tokio::test] - async fn test_recall_measurement_u16() { - let datum = Datum::U16(1); - let as_json = serde_json::Value::from(1_u16); - test_recall_measurement_impl::(datum, None, as_json).await; - } + // However, there's only 1 i64 field, `cpu_id`. + assert_table_count(&client, "fields_i64", 8).await; - #[tokio::test] - async fn test_recall_measurement_i32() { - let datum = Datum::I32(1); - let as_json = serde_json::Value::from(1_i32); - test_recall_measurement_impl::(datum, None, as_json).await; + assert_table_count( + &client, + "measurements_cumulativef64", + samples.len(), + ) + .await; + + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_recall_measurement_u32() { - let datum = Datum::U32(1); - let as_json = serde_json::Value::from(1_u32); - test_recall_measurement_impl::(datum, None, as_json).await; - } + // Regression test verifying that integers are returned in the expected format from the + // database. + // + // By default, ClickHouse _quotes_ 64-bit integers, which is apparently to support JavaScript + // implementations of JSON. See https://github.com/ClickHouse/ClickHouse/issues/2375 for + // details. This test verifies that we get back _unquoted_ integers from the database. + async fn unquoted_64bit_integers_test( + address: SocketAddr, + ) -> Result<(), Error> { + use serde_json::Value; + let logctx = test_setup_log("test_unquoted_64bit_integers"); + let log = &logctx.log; - #[tokio::test] - async fn test_recall_measurement_i64() { - let datum = Datum::I64(1); - let as_json = serde_json::Value::from(1_i64); - test_recall_measurement_impl::(datum, None, as_json).await; - } + let client = Client::new(address, &log); + client + .init_single_node_db() + .await + .expect("Failed to initialize timeseries database"); + let output = client + .execute_with_body( + "SELECT toUInt64(1) AS foo FORMAT JSONEachRow;".to_string(), + ) + .await + .unwrap(); + let json: Value = serde_json::from_str(&output).unwrap(); + assert_eq!(json["foo"], Value::Number(1u64.into())); - #[tokio::test] - async fn test_recall_measurement_u64() { - let datum = Datum::U64(1); - let as_json = serde_json::Value::from(1_u64); - test_recall_measurement_impl::(datum, None, as_json).await; + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_recall_measurement_f32() { - const VALUE: f32 = 1.1; - let datum = Datum::F32(VALUE); - // NOTE: This is intentionally an f64. - let as_json = serde_json::Value::from(1.1_f64); - test_recall_measurement_impl::(datum, None, as_json).await; - } + async fn differentiate_by_timeseries_name_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_differentiate_by_timeseries_name"); + let log = &logctx.log; - #[tokio::test] - async fn test_recall_measurement_f64() { - const VALUE: f64 = 1.1; - let datum = Datum::F64(VALUE); - let as_json = serde_json::Value::from(VALUE); - test_recall_measurement_impl::(datum, None, as_json).await; - } + #[derive(Debug, Default, PartialEq, oximeter::Target)] + struct MyTarget { + id: i64, + } - #[tokio::test] - async fn test_recall_measurement_cumulative_i64() { - let datum = Datum::CumulativeI64(1.into()); - let as_json = serde_json::Value::from(1_i64); - test_recall_measurement_impl::(datum, None, as_json).await; - } + // These two metrics share a target and have no fields. Thus they have the same timeseries + // keys. This test is to verify we can distinguish between them, which relies on their + // names. + #[derive(Debug, Default, PartialEq, oximeter::Metric)] + struct FirstMetric { + datum: i64, + } - #[tokio::test] - async fn test_recall_measurement_cumulative_u64() { - let datum = Datum::CumulativeU64(1.into()); - let as_json = serde_json::Value::from(1_u64); - test_recall_measurement_impl::(datum, None, as_json).await; - } + #[derive(Debug, Default, PartialEq, oximeter::Metric)] + struct SecondMetric { + datum: i64, + } - #[tokio::test] - async fn test_recall_measurement_cumulative_f64() { - let datum = Datum::CumulativeF64(1.1.into()); - let as_json = serde_json::Value::from(1.1_f64); - test_recall_measurement_impl::(datum, None, as_json).await; - } + let client = Client::new(address, &log); + client + .init_single_node_db() + .await + .expect("Failed to initialize timeseries database"); - async fn histogram_test_impl(hist: Histogram) - where - T: oximeter::histogram::HistogramSupport, - Datum: From>, - serde_json::Value: From, - { - let (bins, counts) = hist.to_arrays(); - let datum = Datum::from(hist); - let as_json = serde_json::Value::Array( - counts.into_iter().map(Into::into).collect(), - ); - test_recall_measurement_impl(datum, Some(bins), as_json).await; - } + let target = MyTarget::default(); + let first_metric = FirstMetric::default(); + let second_metric = SecondMetric::default(); - #[tokio::test] - async fn test_recall_measurement_histogram_i8() { - let hist = Histogram::new(&[0i8, 1, 2]).unwrap(); - histogram_test_impl(hist).await; - } + let samples = &[ + Sample::new(&target, &first_metric).unwrap(), + Sample::new(&target, &second_metric).unwrap(), + ]; + client + .insert_samples(samples) + .await + .expect("Failed to insert test samples"); - #[tokio::test] - async fn test_recall_measurement_histogram_u8() { - let hist = Histogram::new(&[0u8, 1, 2]).unwrap(); - histogram_test_impl(hist).await; - } + let results = client + .select_timeseries_with( + "my_target:second_metric", + &["id==0"], + None, + None, + None, + None, + ) + .await + .expect("Failed to select test samples"); + println!("{:#?}", results); + assert_eq!(results.len(), 1, "Expected only one timeseries"); + let timeseries = &results[0]; + assert_eq!( + timeseries.measurements.len(), + 1, + "Expected only one sample" + ); + assert_eq!(timeseries.target.name, "my_target"); + assert_eq!(timeseries.metric.name, "second_metric"); - #[tokio::test] - async fn test_recall_measurement_histogram_i16() { - let hist = Histogram::new(&[0i16, 1, 2]).unwrap(); - histogram_test_impl(hist).await; + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_recall_measurement_histogram_u16() { - let hist = Histogram::new(&[0u16, 1, 2]).unwrap(); - histogram_test_impl(hist).await; - } + async fn select_timeseries_with_select_one_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_select_timeseries_with_select_one"); + let log = &logctx.log; - #[tokio::test] - async fn test_recall_measurement_histogram_i32() { - let hist = Histogram::new(&[0i32, 1, 2]).unwrap(); - histogram_test_impl(hist).await; - } + let (target, metrics, samples) = setup_select_test(); - #[tokio::test] - async fn test_recall_measurement_histogram_u32() { - let hist = Histogram::new(&[0u32, 1, 2]).unwrap(); - histogram_test_impl(hist).await; - } + let client = Client::new(address, &log); + client + .init_single_node_db() + .await + .expect("Failed to initialize timeseries database"); + client + .insert_samples(&samples) + .await + .expect("Failed to insert samples"); - #[tokio::test] - async fn test_recall_measurement_histogram_i64() { - let hist = Histogram::new(&[0i64, 1, 2]).unwrap(); - histogram_test_impl(hist).await; - } + let timeseries_name = "service:request_latency"; + // This set of criteria should select exactly one timeseries, with two measurements. + // The target is the same in all cases, but we're looking for the first of the metrics, and + // the first two samples/measurements. + let criteria = + &["name==oximeter", "route==/a", "method==GET", "status_code==200"]; + let mut timeseries = client + .select_timeseries_with( + timeseries_name, + criteria, + None, + None, + None, + None, + ) + .await + .expect("Failed to select timeseries"); - #[tokio::test] - async fn test_recall_measurement_histogram_u64() { - let hist = Histogram::new(&[0u64, 1, 2]).unwrap(); - histogram_test_impl(hist).await; - } + // NOTE: Timeseries as returned from the database are sorted by (name, key, timestamp). + // However, that key is a hash of the field values, which effectively randomizes each + // timeseries with the same name relative to one another. Resort them here, so that the + // timeseries are in ascending order of first timestamp, so that we can reliably test them. + timeseries.sort_by(|first, second| { + first.measurements[0] + .timestamp() + .cmp(&second.measurements[0].timestamp()) + }); - // NOTE: This test is ignored intentionally. - // - // We're using the JSONEachRow format to return data, which loses precision - // for floating point values. This means we return the _double_ 0.1 from - // the database as a `Value::Number`, which fails to compare equal to the - // `Value::Number(0.1f32 as f64)` we sent in. That's because 0.1 is not - // exactly representable in an `f32`, but it's close enough that ClickHouse - // prints `0.1` in the result, which converts to a slightly different `f64` - // than `0.1_f32 as f64` does. - // - // See https://github.com/oxidecomputer/omicron/issues/4059 for related - // discussion. - #[tokio::test] - #[ignore] - async fn test_recall_measurement_histogram_f32() { - let hist = Histogram::new(&[0.1f32, 0.2, 0.3]).unwrap(); - histogram_test_impl(hist).await; - } + assert_eq!(timeseries.len(), 1, "Expected one timeseries"); + let timeseries = timeseries.get(0).unwrap(); + assert_eq!( + timeseries.measurements.len(), + 2, + "Expected exactly two measurements" + ); + verify_measurements(×eries.measurements, &samples[..2]); + verify_target(×eries.target, &target); + verify_metric(×eries.metric, metrics.get(0).unwrap()); - #[tokio::test] - async fn test_recall_measurement_histogram_f64() { - let hist = Histogram::new(&[0.1f64, 0.2, 0.3]).unwrap(); - histogram_test_impl(hist).await; + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) } - async fn test_recall_measurement_impl + Copy>( - datum: Datum, - maybe_bins: Option>, - json_datum: serde_json::Value, - ) { + async fn select_timeseries_with_select_one_field_with_multiple_values_test( + address: SocketAddr, + ) -> Result<(), Error> { let logctx = test_setup_log( - format!("test_recall_measurement_{}", datum.datum_type()).as_str(), + "test_select_timeseries_with_select_one_field_with_multiple_values", ); let log = &logctx.log; - // Let the OS assign a port and discover it after ClickHouse starts - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); + let (target, metrics, samples) = setup_select_test(); - let client = Client::new(address, log); + let client = Client::new(address, &log); client .init_single_node_db() .await .expect("Failed to initialize timeseries database"); + client + .insert_samples(&samples) + .await + .expect("Failed to insert samples"); - // Insert a record from this datum. - const TIMESERIES_NAME: &str = "foo:bar"; - const TIMESERIES_KEY: u64 = 101; - let mut inserted_row = serde_json::Map::new(); - inserted_row - .insert("timeseries_name".to_string(), TIMESERIES_NAME.into()); - inserted_row - .insert("timeseries_key".to_string(), TIMESERIES_KEY.into()); - inserted_row.insert( - "timestamp".to_string(), - Utc::now() - .format(crate::DATABASE_TIMESTAMP_FORMAT) - .to_string() - .into(), - ); - - // Insert the start time and possibly bins. - if let Some(start_time) = datum.start_time() { - inserted_row.insert( - "start_time".to_string(), - start_time - .format(crate::DATABASE_TIMESTAMP_FORMAT) - .to_string() - .into(), - ); - } - if let Some(bins) = &maybe_bins { - let bins = serde_json::Value::Array( - bins.iter().copied().map(Into::into).collect(), + let timeseries_name = "service:request_latency"; + // This set of criteria should select the last two metrics, and so the last two + // timeseries. The target is the same in all cases. + let criteria = + &["name==oximeter", "route==/a", "method==GET", "status_code>200"]; + let mut timeseries = client + .select_timeseries_with( + timeseries_name, + criteria, + None, + None, + None, + None, + ) + .await + .expect("Failed to select timeseries"); + + timeseries.sort_by(|first, second| { + first.measurements[0] + .timestamp() + .cmp(&second.measurements[0].timestamp()) + }); + + assert_eq!(timeseries.len(), 2, "Expected two timeseries"); + for (i, ts) in timeseries.iter().enumerate() { + assert_eq!( + ts.measurements.len(), + 2, + "Expected exactly two measurements" ); - inserted_row.insert("bins".to_string(), bins); - inserted_row.insert("counts".to_string(), json_datum); - } else { - inserted_row.insert("datum".to_string(), json_datum); + + // Metrics 1..3 in the third axis, status code. + let sample_start = unravel_index(&[0, 0, i + 1, 0]); + let sample_end = sample_start + 2; + verify_measurements( + &ts.measurements, + &samples[sample_start..sample_end], + ); + verify_target(&ts.target, &target); } - let inserted_row = serde_json::Value::from(inserted_row); - let measurement_table = measurement_table_name(datum.datum_type()); - let row = serde_json::to_string(&inserted_row).unwrap(); - let insert_sql = format!( - "INSERT INTO oximeter.{measurement_table} FORMAT JSONEachRow {row}", - ); - client - .execute(insert_sql) - .await - .expect("Failed to insert measurement row"); + for (ts, metric) in timeseries.iter().zip(metrics[1..3].iter()) { + verify_metric(&ts.metric, metric); + } - // Select it exactly back out. - let select_sql = format!( - "SELECT * FROM oximeter.{} LIMIT 1 FORMAT {};", - measurement_table, - crate::DATABASE_SELECT_FORMAT, - ); - let body = client - .execute_with_body(select_sql) - .await - .expect("Failed to select measurement row"); - let actual_row: serde_json::Value = serde_json::from_str(&body) - .expect("Failed to parse measurement row JSON"); - println!("{actual_row:?}"); - println!("{inserted_row:?}"); - assert_eq!( - actual_row, inserted_row, - "Actual and expected measurement rows do not match" - ); - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_schema_mismatch() { - let logctx = test_setup_log("test_schema_mismatch"); + async fn select_timeseries_with_select_multiple_fields_with_multiple_values_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_select_timeseries_with_select_multiple_fields_with_multiple_values"); let log = &logctx.log; - // Let the OS assign a port and discover it after ClickHouse starts - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); + let (target, metrics, samples) = setup_select_test(); let client = Client::new(address, &log); client .init_single_node_db() .await .expect("Failed to initialize timeseries database"); - let sample = test_util::make_sample(); - client.insert_samples(&[sample]).await.unwrap(); - - let bad_name = name_mismatch::TestTarget { - name: "first_name".into(), - name2: "second_name".into(), - num: 2, - }; - let metric = test_util::TestMetric { - id: uuid::Uuid::new_v4(), - good: true, - datum: 1, - }; - let sample = Sample::new(&bad_name, &metric).unwrap(); - let result = client.verify_sample_schema(&sample).await; - assert!(matches!(result, Err(Error::SchemaMismatch { .. }))); - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); - logctx.cleanup_successful(); - } - - // Returns the number of timeseries schemas being used. - async fn get_schema_count(client: &Client) -> usize { client - .execute_with_body( - "SELECT * FROM oximeter.timeseries_schema FORMAT JSONEachRow;", - ) + .insert_samples(&samples) .await - .expect("Failed to SELECT from database") - .lines() - .count() - } - - #[tokio::test] - async fn test_database_version_update_idempotent() { - let logctx = test_setup_log("test_database_version_update_idempotent"); - let log = &logctx.log; + .expect("Failed to insert samples"); - let mut db = ClickHouseInstance::new_single_node(0) + let timeseries_name = "service:request_latency"; + // This is non-selective for the route, which is the "second axis", and has two values for + // the third axis, status code. There should be a total of 4 timeseries, since there are + // two methods and two possible status codes. + let criteria = &["name==oximeter", "route==/a", "status_code>200"]; + let mut timeseries = client + .select_timeseries_with( + timeseries_name, + criteria, + None, + None, + None, + None, + ) .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - - let replicated = false; + .expect("Failed to select timeseries"); - // Initialize the database... - let client = Client::new(address, &log); - client - .initialize_db_with_version(replicated, model::OXIMETER_VERSION) - .await - .expect("Failed to initialize timeseries database"); + timeseries.sort_by(|first, second| { + first.measurements[0] + .timestamp() + .cmp(&second.measurements[0].timestamp()) + }); - // Insert data here so we can verify it still exists later. - // - // The values here don't matter much, we just want to check that - // the database data hasn't been dropped. - assert_eq!(0, get_schema_count(&client).await); - let sample = test_util::make_sample(); - client.insert_samples(&[sample.clone()]).await.unwrap(); - assert_eq!(1, get_schema_count(&client).await); + assert_eq!(timeseries.len(), 4, "Expected four timeseries"); + let indices = &[(0, 1), (0, 2), (1, 1), (1, 2)]; + for (i, ts) in timeseries.iter().enumerate() { + assert_eq!( + ts.measurements.len(), + 2, + "Expected exactly two measurements" + ); - // Re-initialize the database, see that our data still exists - client - .initialize_db_with_version(replicated, model::OXIMETER_VERSION) - .await - .expect("Failed to initialize timeseries database"); + // Metrics 0..2 in the second axis, method + // Metrics 1..3 in the third axis, status code. + let (i0, i1) = indices[i]; + let sample_start = unravel_index(&[0, i0, i1, 0]); + let sample_end = sample_start + 2; + verify_measurements( + &ts.measurements, + &samples[sample_start..sample_end], + ); + verify_target(&ts.target, &target); + } - assert_eq!(1, get_schema_count(&client).await); + let mut ts_iter = timeseries.iter(); + for i in 0..2 { + for j in 1..3 { + let ts = ts_iter.next().unwrap(); + let metric = metrics.get(i * 3 + j).unwrap(); + verify_metric(&ts.metric, metric); + } + } - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_database_version_will_not_downgrade() { - let logctx = test_setup_log("test_database_version_will_not_downgrade"); + async fn select_timeseries_with_all_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_select_timeseries_with_all"); let log = &logctx.log; - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - - let replicated = false; + let (target, metrics, samples) = setup_select_test(); - // Initialize the database let client = Client::new(address, &log); client - .initialize_db_with_version(replicated, model::OXIMETER_VERSION) + .init_single_node_db() .await .expect("Failed to initialize timeseries database"); - - // Bump the version of the database to a "too new" version client - .insert_version(model::OXIMETER_VERSION + 1) + .insert_samples(&samples) .await - .expect("Failed to insert very new DB version"); + .expect("Failed to insert samples"); - // Expect a failure re-initializing the client. - // - // This will attempt to initialize the client with "version = - // model::OXIMETER_VERSION", which is "too old". - client - .initialize_db_with_version(replicated, model::OXIMETER_VERSION) + let timeseries_name = "service:request_latency"; + let mut timeseries = client + .select_timeseries_with( + timeseries_name, + // We're selecting all timeseries/samples here. + &[], + None, + None, + None, + None, + ) .await - .expect_err("Should have failed, downgrades are not supported"); + .expect("Failed to select timeseries"); - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + timeseries.sort_by(|first, second| { + first.measurements[0] + .timestamp() + .cmp(&second.measurements[0].timestamp()) + }); + + assert_eq!(timeseries.len(), 12, "Expected 12 timeseries"); + for (i, ts) in timeseries.iter().enumerate() { + assert_eq!( + ts.measurements.len(), + 2, + "Expected exactly two measurements" + ); + + let sample_start = i * 2; + let sample_end = sample_start + 2; + verify_measurements( + &ts.measurements, + &samples[sample_start..sample_end], + ); + verify_target(&ts.target, &target); + verify_metric(&ts.metric, metrics.get(i).unwrap()); + } + + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_database_version_wipes_old_version() { - let logctx = test_setup_log("test_database_version_wipes_old_version"); + async fn select_timeseries_with_start_time_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_select_timeseries_with_start_time"); let log = &logctx.log; - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let replicated = false; + let (_, metrics, samples) = setup_select_test(); - // Initialize the Client let client = Client::new(address, &log); client - .initialize_db_with_version(replicated, model::OXIMETER_VERSION) + .init_single_node_db() .await .expect("Failed to initialize timeseries database"); - - // Insert data here so we can remove it later. - // - // The values here don't matter much, we just want to check that - // the database data gets dropped later. - assert_eq!(0, get_schema_count(&client).await); - let sample = test_util::make_sample(); - client.insert_samples(&[sample.clone()]).await.unwrap(); - assert_eq!(1, get_schema_count(&client).await); - - // If we try to upgrade to a newer version, we'll drop old data. client - .initialize_db_with_version(replicated, model::OXIMETER_VERSION + 1) + .insert_samples(&samples) .await - .expect("Should have initialized database successfully"); - assert_eq!(0, get_schema_count(&client).await); - - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn test_schema_update() { - let logctx = test_setup_log("test_schema_update"); - let log = &logctx.log; + .expect("Failed to insert samples"); - // Let the OS assign a port and discover it after ClickHouse starts - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); + let timeseries_name = "service:request_latency"; + let start_time = samples[samples.len() / 2].measurement.timestamp(); + let mut timeseries = client + .select_timeseries_with( + timeseries_name, + // We're selecting all timeseries/samples here. + &[], + Some(query::Timestamp::Exclusive(start_time)), + None, + None, + None, + ) + .await + .expect("Failed to select timeseries"); + + timeseries.sort_by(|first, second| { + first.measurements[0] + .timestamp() + .cmp(&second.measurements[0].timestamp()) + }); + + assert_eq!(timeseries.len(), metrics.len() / 2); + for ts in timeseries.iter() { + for meas in ts.measurements.iter() { + assert!(meas.timestamp() > start_time); + } + } + + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) + } + async fn select_timeseries_with_limit_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_select_timeseries_with_limit"); + let log = &logctx.log; + + let (_, _, samples) = setup_select_test(); let client = Client::new(address, &log); client .init_single_node_db() .await .expect("Failed to initialize timeseries database"); - let sample = test_util::make_sample(); + client + .insert_samples(&samples) + .await + .expect("Failed to insert samples"); - // Verify that this sample is considered new, i.e., we return rows to update the timeseries - // schema table. - let result = client.verify_sample_schema(&sample).await.unwrap(); - assert!( - matches!(result, Some(_)), - "When verifying a new sample, the rows to be inserted should be returned" - ); + let timeseries_name = "service:request_latency"; + // We have to define criteria that resolve to a single timeseries + let criteria = + &["name==oximeter", "route==/a", "method==GET", "status_code==200"]; + // First, query without a limit. We should see all the results. + let all_measurements = &client + .select_timeseries_with( + timeseries_name, + criteria, + None, + None, + None, + None, + ) + .await + .expect("Failed to select timeseries")[0] + .measurements; - // Clear the internal caches of seen schema - client.schema.lock().unwrap().clear(); + // Check some constraints on the number of measurements - we + // can change these, but these assumptions make the test simpler. + // + // For now, assume we can cleanly cut the number of measurements in + // half. + assert!(all_measurements.len() >= 2); + assert!(all_measurements.len() % 2 == 0); - // Insert the new sample - client.insert_samples(&[sample.clone()]).await.unwrap(); + // Next, let's set a limit to half the results and query again. + let limit = + NonZeroU32::new(u32::try_from(all_measurements.len() / 2).unwrap()) + .unwrap(); - // The internal map should now contain both the new timeseries schema - let actual_schema = model::schema_for(&sample); - let timeseries_name = - TimeseriesName::try_from(sample.timeseries_name.as_str()).unwrap(); - let expected_schema = client - .schema - .lock() - .unwrap() - .get(×eries_name) - .expect( - "After inserting a new sample, its schema should be included", + // We expect queries with a limit to fail when they fail to resolve to a + // single timeseries, so run a query without any criteria to test that + client + .select_timeseries_with( + timeseries_name, + &[], + None, + None, + Some(limit), + None, ) - .clone(); - assert_eq!( - actual_schema, - expected_schema, - "The timeseries schema for a new sample was not correctly inserted into internal cache", - ); + .await + .expect_err("Should fail to select timeseries"); - // This should no longer return a new row to be inserted for the schema of this sample, as - // any schema have been included above. - let result = client.verify_sample_schema(&sample).await.unwrap(); - assert!( - matches!(result, None), - "After inserting new schema, it should no longer be considered new" + // queries with limit do not fail when they resolve to zero timeseries + let empty_result = &client + .select_timeseries_with( + timeseries_name, + &["name==not_a_real_name"], + None, + None, + Some(limit), + None, + ) + .await + .expect("Failed to select timeseries"); + assert_eq!(empty_result.len(), 0); + + let timeseries = &client + .select_timeseries_with( + timeseries_name, + criteria, + None, + None, + Some(limit), + None, + ) + .await + .expect("Failed to select timeseries")[0]; + assert_eq!(timeseries.measurements.len() as u32, limit.get()); + assert_eq!( + all_measurements[..all_measurements.len() / 2], + timeseries.measurements ); - // Verify that it's actually in the database! - let sql = String::from( - "SELECT * FROM oximeter.timeseries_schema FORMAT JSONEachRow;", + // Get the other half of the results. + let timeseries = &client + .select_timeseries_with( + timeseries_name, + criteria, + Some(query::Timestamp::Exclusive( + timeseries.measurements.last().unwrap().timestamp(), + )), + None, + Some(limit), + None, + ) + .await + .expect("Failed to select timeseries")[0]; + assert_eq!(timeseries.measurements.len() as u32, limit.get()); + assert_eq!( + all_measurements[all_measurements.len() / 2..], + timeseries.measurements ); - let result = client.execute_with_body(sql).await.unwrap(); - let schema = result - .lines() - .map(|line| { - TimeseriesSchema::from( - serde_json::from_str::(&line) - .unwrap(), - ) - }) - .collect::>(); - assert_eq!(schema.len(), 1); - assert_eq!(expected_schema, schema[0]); - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } - async fn setup_filter_testcase() -> (ClickHouseInstance, Client, Vec) - { - let log = slog::Logger::root(slog_dtrace::Dtrace::new().0, o!()); - - // Let the OS assign a port and discover it after ClickHouse starts - let db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); + async fn select_timeseries_with_order_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_select_timeseries_with_order"); + let log = &logctx.log; + let (_, _, samples) = setup_select_test(); let client = Client::new(address, &log); client .init_single_node_db() .await .expect("Failed to initialize timeseries database"); + client + .insert_samples(&samples) + .await + .expect("Failed to insert samples"); - // Create sample data - let (n_projects, n_instances, n_cpus, n_samples) = (2, 2, 2, 2); - let samples = test_util::generate_test_samples( - n_projects, - n_instances, - n_cpus, - n_samples, - ); - assert_eq!( - samples.len(), - n_projects * n_instances * n_cpus * n_samples - ); - - client.insert_samples(&samples).await.unwrap(); - (db, client, samples) - } - - #[tokio::test] - async fn test_client_select_timeseries_one() { - let (mut db, client, samples) = setup_filter_testcase().await; - let sample = samples.first().unwrap(); - let target_fields = sample.target_fields().collect::>(); - let metric_fields = sample.metric_fields().collect::>(); - let criteria = &[ - format!( - "project_id=={}", - target_fields - .iter() - .find(|f| f.name == "project_id") - .unwrap() - .value - ), - format!( - "instance_id=={}", - target_fields - .iter() - .find(|f| f.name == "instance_id") - .unwrap() - .value - ), - format!("cpu_id=={}", metric_fields[0].value), - ]; - let results = client + let timeseries_name = "service:request_latency"; + // Limits only work with a single timeseries, so we have to use criteria + // that resolve to a single timeseries + let criteria = + &["name==oximeter", "route==/a", "method==GET", "status_code==200"]; + // First, query without an order. We should see all the results in ascending order. + let all_measurements = &client .select_timeseries_with( - &sample.timeseries_name, - &criteria.iter().map(|x| x.as_str()).collect::>(), + timeseries_name, + criteria, None, None, None, None, ) .await - .unwrap(); - assert_eq!(results.len(), 1, "Expected to find a single timeseries"); - let timeseries = &results[0]; - assert_eq!( - timeseries.measurements.len(), - 2, - "Expected 2 samples per timeseries" + .expect("Failed to select timeseries")[0] + .measurements; + + assert!( + all_measurements.len() > 1, + "need more than one measurement to test ordering" ); - // Compare measurements themselves - let expected_measurements = - samples.iter().map(|sample| &sample.measurement); - let actual_measurements = timeseries.measurements.iter(); - assert!(actual_measurements - .zip(expected_measurements) - .all(|(first, second)| first == second)); - assert_eq!(timeseries.target.name, "virtual_machine"); - // Compare fields, but order might be different. - fn field_cmp<'a>( - needle: &'a crate::Field, - mut haystack: impl Iterator, - ) -> bool { - needle == haystack.find(|f| f.name == needle.name).unwrap() - } - timeseries - .target - .fields - .iter() - .all(|field| field_cmp(field, sample.target_fields())); - assert_eq!(timeseries.metric.name, "cpu_busy"); - timeseries - .metric - .fields - .iter() - .all(|field| field_cmp(field, sample.metric_fields())); - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); - } - - #[tokio::test] - async fn test_field_record_count() { - // This test verifies that the number of records in the field tables is as expected. - // - // Because of the schema change, inserting field records per field per unique timeseries, - // we'd like to exercise the logic of ClickHouse's replacing merge tree engine. - let (mut db, client, samples) = setup_filter_testcase().await; - - async fn assert_table_count( - client: &Client, - table: &str, - expected_count: usize, - ) { - let body = client - .execute_with_body(format!( - "SELECT COUNT() FROM oximeter.{};", - table - )) - .await - .unwrap(); - let actual_count = - body.lines().next().unwrap().trim().parse::().expect( - "Expected a count of the number of rows from ClickHouse", - ); - assert_eq!(actual_count, expected_count); - } + // Explicitly specifying asc should give the same results + let timeseries_asc = &client + .select_timeseries_with( + timeseries_name, + criteria, + None, + None, + None, + Some(PaginationOrder::Ascending), + ) + .await + .expect("Failed to select timeseries")[0] + .measurements; + assert_eq!(all_measurements, timeseries_asc); - // There should be (2 projects * 2 instances * 2 cpus) == 8 timeseries. For each of these - // timeseries, there are 2 UUID fields, `project_id` and `instance_id`. So 16 UUID records. - assert_table_count(&client, "fields_uuid", 16).await; + // Now get the results in reverse order + let timeseries_desc = &client + .select_timeseries_with( + timeseries_name, + criteria, + None, + None, + None, + Some(PaginationOrder::Descending), + ) + .await + .expect("Failed to select timeseries")[0] + .measurements; - // However, there's only 1 i64 field, `cpu_id`. - assert_table_count(&client, "fields_i64", 8).await; + let mut timeseries_asc_rev = timeseries_asc.clone(); + timeseries_asc_rev.reverse(); - assert_table_count( - &client, - "measurements_cumulativef64", - samples.len(), - ) - .await; - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); - } + assert_ne!(timeseries_desc, timeseries_asc); + assert_eq!(timeseries_desc, ×eries_asc_rev); - // Regression test verifying that integers are returned in the expected format from the - // database. - // - // By default, ClickHouse _quotes_ 64-bit integers, which is apparently to support JavaScript - // implementations of JSON. See https://github.com/ClickHouse/ClickHouse/issues/2375 for - // details. This test verifies that we get back _unquoted_ integers from the database. - #[tokio::test] - async fn test_unquoted_64bit_integers() { - use serde_json::Value; - let (mut db, client, _) = setup_filter_testcase().await; - let output = client - .execute_with_body( - "SELECT toUInt64(1) AS foo FORMAT JSONEachRow;".to_string(), + // can use limit 1 to get single most recent measurement + let desc_limit_1 = &client + .select_timeseries_with( + timeseries_name, + criteria, + None, + None, + Some(NonZeroU32::new(1).unwrap()), + Some(PaginationOrder::Descending), ) .await - .unwrap(); - let json: Value = serde_json::from_str(&output).unwrap(); - assert_eq!(json["foo"], Value::Number(1u64.into())); - db.cleanup().await.expect("Failed to cleanup ClickHouse server"); - } + .expect("Failed to select timeseries")[0] + .measurements; - #[tokio::test] - async fn test_bad_database_connection() { - let logctx = test_setup_log("test_bad_database_connection"); - let log = &logctx.log; - let client = Client::new("127.0.0.1:443".parse().unwrap(), &log); - assert!(matches!( - client.ping().await, - Err(Error::DatabaseUnavailable(_)) - )); + assert_eq!(desc_limit_1.len(), 1); + assert_eq!( + desc_limit_1.first().unwrap(), + timeseries_asc.last().unwrap() + ); + + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_differentiate_by_timeseries_name() { - #[derive(Debug, Default, PartialEq, oximeter::Target)] - struct MyTarget { - id: i64, - } - - // These two metrics share a target and have no fields. Thus they have the same timeseries - // keys. This test is to verify we can distinguish between them, which relies on their - // names. - #[derive(Debug, Default, PartialEq, oximeter::Metric)] - struct FirstMetric { - datum: i64, - } - - #[derive(Debug, Default, PartialEq, oximeter::Metric)] - struct SecondMetric { - datum: i64, - } - - let logctx = test_setup_log("test_differentiate_by_timeseries_name"); + async fn get_schema_no_new_values_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_get_schema_no_new_values"); let log = &logctx.log; - // Let the OS assign a port and discover it after ClickHouse starts - let db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let client = Client::new(address, &log); client .init_single_node_db() .await .expect("Failed to initialize timeseries database"); + let samples = test_util::generate_test_samples(2, 2, 2, 2); + client.insert_samples(&samples).await?; - let target = MyTarget::default(); - let first_metric = FirstMetric::default(); - let second_metric = SecondMetric::default(); - - let samples = &[ - Sample::new(&target, &first_metric).unwrap(), - Sample::new(&target, &second_metric).unwrap(), - ]; + let original_schema = client.schema.lock().await.clone(); + let mut schema = client.schema.lock().await; client - .insert_samples(samples) + .get_schema_locked(&mut schema) .await - .expect("Failed to insert test samples"); + .expect("Failed to get timeseries schema"); + assert_eq!(&original_schema, &*schema, "Schema shouldn't change"); - let results = client - .select_timeseries_with( - "my_target:second_metric", - &["id==0"], - None, - None, - None, - None, - ) - .await - .expect("Failed to select test samples"); - println!("{:#?}", results); - assert_eq!(results.len(), 1, "Expected only one timeseries"); - let timeseries = &results[0]; - assert_eq!( - timeseries.measurements.len(), - 1, - "Expected only one sample" - ); - assert_eq!(timeseries.target.name, "my_target"); - assert_eq!(timeseries.metric.name, "second_metric"); + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } - #[derive(Debug, Clone, oximeter::Target)] - struct Service { - name: String, - id: uuid::Uuid, - } - #[derive(Debug, Clone, oximeter::Metric)] - struct RequestLatency { - route: String, - method: String, - status_code: i64, - #[datum] - latency: f64, - } - - const SELECT_TEST_ID: &str = "4fa827ea-38bb-c37e-ac2d-f8432ca9c76e"; - fn setup_select_test() -> (Service, Vec, Vec) { - // One target - let id = SELECT_TEST_ID.parse().unwrap(); - let target = Service { name: "oximeter".to_string(), id }; - - // Many metrics - let routes = &["/a", "/b"]; - let methods = &["GET", "POST"]; - let status_codes = &[200, 204, 500]; - - // Two samples each - let n_timeseries = routes.len() * methods.len() * status_codes.len(); - let mut metrics = Vec::with_capacity(n_timeseries); - let mut samples = Vec::with_capacity(n_timeseries * 2); - for (route, method, status_code) in - itertools::iproduct!(routes, methods, status_codes) - { - let metric = RequestLatency { - route: route.to_string(), - method: method.to_string(), - status_code: *status_code, - latency: 0.0, - }; - samples.push(Sample::new(&target, &metric).unwrap()); - samples.push(Sample::new(&target, &metric).unwrap()); - metrics.push(metric); - } - (target, metrics, samples) - } + async fn timeseries_schema_list_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_timeseries_schema_list"); + let log = &logctx.log; - async fn test_select_timeseries_with_impl( - criteria: &[&str], - start_time: Option, - end_time: Option, - test_fn: impl Fn(&Service, &[RequestLatency], &[Sample], &[Timeseries]), - ) { - let (target, metrics, samples) = setup_select_test(); - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let log = Logger::root(slog::Discard, o!()); let client = Client::new(address, &log); client .init_single_node_db() .await .expect("Failed to initialize timeseries database"); - client - .insert_samples(&samples) - .await - .expect("Failed to insert samples"); - let timeseries_name = "service:request_latency"; - let mut timeseries = client - .select_timeseries_with( - timeseries_name, - criteria, - start_time, - end_time, - None, - None, - ) - .await - .expect("Failed to select timeseries"); + let samples = test_util::generate_test_samples(2, 2, 2, 2); + client.insert_samples(&samples).await?; - // NOTE: Timeseries as returned from the database are sorted by (name, key, timestamp). - // However, that key is a hash of the field values, which effectively randomizes each - // timeseries with the same name relative to one another. Resort them here, so that the - // timeseries are in ascending order of first timestamp, so that we can reliably test them. - timeseries.sort_by(|first, second| { - first.measurements[0] - .timestamp() - .cmp(&second.measurements[0].timestamp()) - }); + let limit = 100u32.try_into().unwrap(); + let page = dropshot::WhichPage::First(dropshot::EmptyScanParams {}); + let result = client.timeseries_schema_list(&page, limit).await.unwrap(); + assert!( + result.items.len() == 1, + "Expected exactly 1 timeseries schema" + ); + let last_seen = result.items.last().unwrap().timeseries_name.clone(); + let page = dropshot::WhichPage::Next(last_seen); + let result = client.timeseries_schema_list(&page, limit).await.unwrap(); + assert!( + result.items.is_empty(), + "Expected the next page of schema to be empty" + ); + assert!( + result.next_page.is_none(), + "Expected the next page token to be None" + ); + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) + } - test_fn(&target, &metrics, &samples, ×eries); + async fn list_timeseries_test(address: SocketAddr) -> Result<(), Error> { + let logctx = test_setup_log("test_list_timeseries"); + let log = &logctx.log; - db.cleanup().await.expect("Failed to cleanup database"); - } + let client = Client::new(address, &log); + client + .init_single_node_db() + .await + .expect("Failed to initialize timeseries database"); + let samples = test_util::generate_test_samples(2, 2, 2, 2); + client.insert_samples(&samples).await?; - // Small helper to go from a mulidimensional index to a flattened array index. - fn unravel_index(idx: &[usize; 4]) -> usize { - let strides = [12, 6, 2, 1]; - let mut index = 0; - for (i, stride) in idx.iter().rev().zip(strides.iter().rev()) { - index += i * stride; - } - index + let limit = 7u32.try_into().unwrap(); + let params = crate::TimeseriesScanParams { + timeseries_name: TimeseriesName::try_from( + "virtual_machine:cpu_busy", + ) + .unwrap(), + criteria: vec!["cpu_id==0".parse().unwrap()], + start_time: None, + end_time: None, + }; + let page = dropshot::WhichPage::First(params.clone()); + let result = client.list_timeseries(&page, limit).await.unwrap(); + + // We should have 4 timeseries, with 2 samples from each of the first 3 and 1 from the last + // timeseries. + assert_eq!(result.items.len(), 4, "Expected 4 timeseries"); + assert_eq!( + result.items.last().unwrap().measurements.len(), + 1, + "Expected 1 sample from the last timeseries" + ); + assert!( + result.items.iter().take(3).all(|ts| ts.measurements.len() == 2), + "Expected 2 samples from the first 3 timeseries" + ); + let last_timeseries = result.items.last().unwrap(); + + // Get the next page. + // + // We have to recreate this as dropshot would, since we cannot build the pagination params + // ourselves. + let next_page = crate::TimeseriesPageSelector { params, offset: limit }; + let page = dropshot::WhichPage::Next(next_page); + let result = client.list_timeseries(&page, limit).await.unwrap(); + + // We should now have the one remaining sample + assert_eq!( + result.items.len(), + 1, + "Expected only 1 timeseries after paginating" + ); + assert_eq!( + result.items[0].measurements.len(), + 1, + "Expected only the last sample after paginating" + ); + assert_eq!( + result.items[0].timeseries_name, last_timeseries.timeseries_name, + "Paginating should pick up where it left off" + ); + assert_eq!( + result.items[0].target, last_timeseries.target, + "Paginating should pick up where it left off" + ); + assert_eq!( + result.items[0].metric, last_timeseries.metric, + "Paginating should pick up where it left off" + ); + + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) } - #[test] - fn test_unravel_index() { - assert_eq!(unravel_index(&[0, 0, 0, 0]), 0); - assert_eq!(unravel_index(&[0, 0, 1, 0]), 2); - assert_eq!(unravel_index(&[1, 0, 0, 0]), 12); - assert_eq!(unravel_index(&[1, 0, 0, 1]), 13); - assert_eq!(unravel_index(&[1, 0, 1, 0]), 14); - assert_eq!(unravel_index(&[1, 1, 2, 1]), 23); + async fn recall_field_value_bool_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::Bool(true); + let as_json = serde_json::Value::from(1_u64); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) } - fn verify_measurements( - measurements: &[oximeter::Measurement], - samples: &[Sample], - ) { - for (measurement, sample) in measurements.iter().zip(samples.iter()) { - assert_eq!( - measurement, &sample.measurement, - "Mismatch between retrieved and expected measurement", - ); - } + async fn recall_field_value_u8_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::U8(1); + let as_json = serde_json::Value::from(1_u8); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) } - fn verify_target(actual: &crate::Target, expected: &Service) { - assert_eq!(actual.name, expected.name()); - for (field_name, field_value) in expected - .field_names() - .into_iter() - .zip(expected.field_values().into_iter()) - { - let actual_field = actual - .fields - .iter() - .find(|f| f.name == *field_name) - .expect("Missing field in recovered timeseries target"); - assert_eq!( - actual_field.value, field_value, - "Incorrect field value in timeseries target" - ); - } + async fn recall_field_value_i8_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::I8(1); + let as_json = serde_json::Value::from(1_i8); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) } - fn verify_metric(actual: &crate::Metric, expected: &RequestLatency) { - assert_eq!(actual.name, expected.name()); - for (field_name, field_value) in expected - .field_names() - .into_iter() - .zip(expected.field_values().into_iter()) - { - let actual_field = actual - .fields - .iter() - .find(|f| f.name == *field_name) - .expect("Missing field in recovered timeseries metric"); - assert_eq!( - actual_field.value, field_value, - "Incorrect field value in timeseries metric" - ); - } + async fn recall_field_value_u16_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::U16(1); + let as_json = serde_json::Value::from(1_u16); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) } - #[tokio::test] - async fn test_select_timeseries_with_select_one() { - // This set of criteria should select exactly one timeseries, with two measurements. - // The target is the same in all cases, but we're looking for the first of the metrics, and - // the first two samples/measurements. - let criteria = - &["name==oximeter", "route==/a", "method==GET", "status_code==200"]; - fn test_fn( - target: &Service, - metrics: &[RequestLatency], - samples: &[Sample], - timeseries: &[Timeseries], - ) { - assert_eq!(timeseries.len(), 1, "Expected one timeseries"); - let timeseries = timeseries.get(0).unwrap(); - assert_eq!( - timeseries.measurements.len(), - 2, - "Expected exactly two measurements" - ); - verify_measurements(×eries.measurements, &samples[..2]); - verify_target(×eries.target, target); - verify_metric(×eries.metric, metrics.get(0).unwrap()); - } - test_select_timeseries_with_impl(criteria, None, None, test_fn).await; + async fn recall_field_value_i16_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::I16(1); + let as_json = serde_json::Value::from(1_i16); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) } - #[tokio::test] - async fn test_select_timeseries_with_select_one_field_with_multiple_values() - { - // This set of criteria should select the last two metrics, and so the last two - // timeseries. The target is the same in all cases. - let criteria = - &["name==oximeter", "route==/a", "method==GET", "status_code>200"]; - fn test_fn( - target: &Service, - metrics: &[RequestLatency], - samples: &[Sample], - timeseries: &[Timeseries], - ) { - assert_eq!(timeseries.len(), 2, "Expected two timeseries"); - for (i, ts) in timeseries.iter().enumerate() { - assert_eq!( - ts.measurements.len(), - 2, - "Expected exactly two measurements" - ); + async fn recall_field_value_u32_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::U32(1); + let as_json = serde_json::Value::from(1_u32); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) + } - // Metrics 1..3 in the third axis, status code. - let sample_start = unravel_index(&[0, 0, i + 1, 0]); - let sample_end = sample_start + 2; - verify_measurements( - &ts.measurements, - &samples[sample_start..sample_end], - ); - verify_target(&ts.target, target); - } + async fn recall_field_value_i32_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::I32(1); + let as_json = serde_json::Value::from(1_i32); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) + } - for (ts, metric) in timeseries.iter().zip(metrics[1..3].iter()) { - verify_metric(&ts.metric, metric); - } - } - test_select_timeseries_with_impl(criteria, None, None, test_fn).await; + async fn recall_field_value_u64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::U64(1); + let as_json = serde_json::Value::from(1_u64); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) } - #[tokio::test] - async fn test_select_timeseries_with_select_multiple_fields_with_multiple_values( - ) { - // This is non-selective for the route, which is the "second axis", and has two values for - // the third axis, status code. There should be a total of 4 timeseries, since there are - // two methods and two possible status codes. - let criteria = &["name==oximeter", "route==/a", "status_code>200"]; - fn test_fn( - target: &Service, - metrics: &[RequestLatency], - samples: &[Sample], - timeseries: &[Timeseries], - ) { - assert_eq!(timeseries.len(), 4, "Expected four timeseries"); - let indices = &[(0, 1), (0, 2), (1, 1), (1, 2)]; - for (i, ts) in timeseries.iter().enumerate() { - assert_eq!( - ts.measurements.len(), - 2, - "Expected exactly two measurements" - ); + async fn recall_field_value_i64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::I64(1); + let as_json = serde_json::Value::from(1_i64); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) + } - // Metrics 0..2 in the second axis, method - // Metrics 1..3 in the third axis, status code. - let (i0, i1) = indices[i]; - let sample_start = unravel_index(&[0, i0, i1, 0]); - let sample_end = sample_start + 2; - verify_measurements( - &ts.measurements, - &samples[sample_start..sample_end], - ); - verify_target(&ts.target, target); - } + async fn recall_field_value_string_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::String("foo".into()); + let as_json = serde_json::Value::from("foo"); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) + } - let mut ts_iter = timeseries.iter(); - for i in 0..2 { - for j in 1..3 { - let ts = ts_iter.next().unwrap(); - let metric = metrics.get(i * 3 + j).unwrap(); - verify_metric(&ts.metric, metric); - } - } - } - test_select_timeseries_with_impl(criteria, None, None, test_fn).await; + async fn recall_field_value_ipv4addr_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::from(Ipv4Addr::LOCALHOST); + let as_json = serde_json::Value::from( + Ipv4Addr::LOCALHOST.to_ipv6_mapped().to_string(), + ); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) } - #[tokio::test] - async fn test_select_timeseries_with_all() { - // We're selecting all timeseries/samples here. - let criteria = &[]; - fn test_fn( - target: &Service, - metrics: &[RequestLatency], - samples: &[Sample], - timeseries: &[Timeseries], - ) { - assert_eq!(timeseries.len(), 12, "Expected 12 timeseries"); - for (i, ts) in timeseries.iter().enumerate() { - assert_eq!( - ts.measurements.len(), - 2, - "Expected exactly two measurements" - ); + async fn recall_field_value_ipv6addr_test( + address: SocketAddr, + ) -> Result<(), Error> { + let field = FieldValue::from(Ipv6Addr::LOCALHOST); + let as_json = serde_json::Value::from(Ipv6Addr::LOCALHOST.to_string()); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) + } - let sample_start = i * 2; - let sample_end = sample_start + 2; - verify_measurements( - &ts.measurements, - &samples[sample_start..sample_end], - ); - verify_target(&ts.target, target); - verify_metric(&ts.metric, metrics.get(i).unwrap()); - } - } - test_select_timeseries_with_impl(criteria, None, None, test_fn).await; + async fn recall_field_value_uuid_test( + address: SocketAddr, + ) -> Result<(), Error> { + let id = Uuid::new_v4(); + let field = FieldValue::from(id); + let as_json = serde_json::Value::from(id.to_string()); + test_recall_field_value_impl(address, field, as_json).await?; + Ok(()) } - #[tokio::test] - async fn test_select_timeseries_with_start_time() { - let (_, metrics, samples) = setup_select_test(); - let mut db = ClickHouseInstance::new_single_node(0) + async fn test_recall_field_value_impl( + address: SocketAddr, + field_value: FieldValue, + as_json: serde_json::Value, + ) -> Result<(), Error> { + let logctx = test_setup_log( + format!("test_recall_field_value_{}", field_value.field_type()) + .as_str(), + ); + let log = &logctx.log; + + let client = Client::new(address, log); + client + .init_single_node_db() .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let logctx = test_setup_log("test_select_timeseries_with_start_time"); + .expect("Failed to initialize timeseries database"); + + // Insert a record from this field. + const TIMESERIES_NAME: &str = "foo:bar"; + const TIMESERIES_KEY: u64 = 101; + const FIELD_NAME: &str = "baz"; + + let mut inserted_row = serde_json::Map::new(); + inserted_row + .insert("timeseries_name".to_string(), TIMESERIES_NAME.into()); + inserted_row + .insert("timeseries_key".to_string(), TIMESERIES_KEY.into()); + inserted_row.insert("field_name".to_string(), FIELD_NAME.into()); + inserted_row.insert("field_value".to_string(), as_json); + let inserted_row = serde_json::Value::from(inserted_row); + + let row = serde_json::to_string(&inserted_row).unwrap(); + let field_table = field_table_name(field_value.field_type()); + let insert_sql = format!( + "INSERT INTO oximeter.{field_table} FORMAT JSONEachRow {row}" + ); + client.execute(insert_sql).await.expect("Failed to insert field row"); + + // Select it exactly back out. + let select_sql = format!( + "SELECT * FROM oximeter.{} LIMIT 1 FORMAT {};", + field_table_name(field_value.field_type()), + crate::DATABASE_SELECT_FORMAT, + ); + let body = client + .execute_with_body(select_sql) + .await + .expect("Failed to select field row"); + let actual_row: serde_json::Value = serde_json::from_str(&body) + .expect("Failed to parse field row JSON"); + println!("{actual_row:?}"); + println!("{inserted_row:?}"); + assert_eq!( + actual_row, inserted_row, + "Actual and expected field rows do not match" + ); + + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) + } + + async fn recall_measurement_bool_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::Bool(true); + let as_json = serde_json::Value::from(1_u64); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_i8_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::I8(1); + let as_json = serde_json::Value::from(1_i8); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_u8_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::U8(1); + let as_json = serde_json::Value::from(1_u8); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_i16_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::I16(1); + let as_json = serde_json::Value::from(1_i16); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_u16_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::U16(1); + let as_json = serde_json::Value::from(1_u16); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_i32_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::I32(1); + let as_json = serde_json::Value::from(1_i32); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_u32_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::U32(1); + let as_json = serde_json::Value::from(1_u32); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_i64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::I64(1); + let as_json = serde_json::Value::from(1_i64); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_u64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::U64(1); + let as_json = serde_json::Value::from(1_u64); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_f32_test( + address: SocketAddr, + ) -> Result<(), Error> { + const VALUE: f32 = 1.1; + let datum = Datum::F32(VALUE); + // NOTE: This is intentionally an f64. + let as_json = serde_json::Value::from(1.1_f64); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_f64_test( + address: SocketAddr, + ) -> Result<(), Error> { + const VALUE: f64 = 1.1; + let datum = Datum::F64(VALUE); + let as_json = serde_json::Value::from(VALUE); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_cumulative_i64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::CumulativeI64(1.into()); + let as_json = serde_json::Value::from(1_i64); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_cumulative_u64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::CumulativeU64(1.into()); + let as_json = serde_json::Value::from(1_u64); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_cumulative_f64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let datum = Datum::CumulativeF64(1.1.into()); + let as_json = serde_json::Value::from(1.1_f64); + test_recall_measurement_impl::(address, datum, None, as_json) + .await?; + Ok(()) + } + + async fn histogram_test_impl( + address: SocketAddr, + hist: Histogram, + ) -> Result<(), Error> + where + T: oximeter::histogram::HistogramSupport, + Datum: From>, + serde_json::Value: From, + { + let (bins, counts) = hist.to_arrays(); + let datum = Datum::from(hist); + let as_json = serde_json::Value::Array( + counts.into_iter().map(Into::into).collect(), + ); + test_recall_measurement_impl(address, datum, Some(bins), as_json) + .await?; + Ok(()) + } + + async fn recall_measurement_histogram_i8_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0i8, 1, 2]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + async fn recall_measurement_histogram_u8_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0u8, 1, 2]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + async fn recall_measurement_histogram_i16_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0i16, 1, 2]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + async fn recall_measurement_histogram_u16_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0u16, 1, 2]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + async fn recall_measurement_histogram_i32_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0i32, 1, 2]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + async fn recall_measurement_histogram_u32_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0u32, 1, 2]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + async fn recall_measurement_histogram_i64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0i64, 1, 2]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + async fn recall_measurement_histogram_u64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0u64, 1, 2]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + // NOTE: This test is ignored intentionally. + // + // We're using the JSONEachRow format to return data, which loses precision + // for floating point values. This means we return the _double_ 0.1 from + // the database as a `Value::Number`, which fails to compare equal to the + // `Value::Number(0.1f32 as f64)` we sent in. That's because 0.1 is not + // exactly representable in an `f32`, but it's close enough that ClickHouse + // prints `0.1` in the result, which converts to a slightly different `f64` + // than `0.1_f32 as f64` does. + // + // See https://github.com/oxidecomputer/omicron/issues/4059 for related + // discussion. + #[allow(dead_code)] + async fn recall_measurement_histogram_f32_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0.1f32, 0.2, 0.3]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + async fn recall_measurement_histogram_f64_test( + address: SocketAddr, + ) -> Result<(), Error> { + let hist = Histogram::new(&[0.1f64, 0.2, 0.3]).unwrap(); + histogram_test_impl(address, hist).await?; + Ok(()) + } + + async fn test_recall_measurement_impl + Copy>( + address: SocketAddr, + datum: Datum, + maybe_bins: Option>, + json_datum: serde_json::Value, + ) -> Result<(), Error> { + let logctx = test_setup_log( + format!("test_recall_measurement_{}", datum.datum_type()).as_str(), + ); + let log = &logctx.log; + + let client = Client::new(address, log); + client + .init_single_node_db() + .await + .expect("Failed to initialize timeseries database"); + + // Insert a record from this datum. + const TIMESERIES_NAME: &str = "foo:bar"; + const TIMESERIES_KEY: u64 = 101; + let mut inserted_row = serde_json::Map::new(); + inserted_row + .insert("timeseries_name".to_string(), TIMESERIES_NAME.into()); + inserted_row + .insert("timeseries_key".to_string(), TIMESERIES_KEY.into()); + inserted_row.insert( + "timestamp".to_string(), + Utc::now() + .format(crate::DATABASE_TIMESTAMP_FORMAT) + .to_string() + .into(), + ); + + // Insert the start time and possibly bins. + if let Some(start_time) = datum.start_time() { + inserted_row.insert( + "start_time".to_string(), + start_time + .format(crate::DATABASE_TIMESTAMP_FORMAT) + .to_string() + .into(), + ); + } + if let Some(bins) = &maybe_bins { + let bins = serde_json::Value::Array( + bins.iter().copied().map(Into::into).collect(), + ); + inserted_row.insert("bins".to_string(), bins); + inserted_row.insert("counts".to_string(), json_datum); + } else { + inserted_row.insert("datum".to_string(), json_datum); + } + let inserted_row = serde_json::Value::from(inserted_row); + + let measurement_table = measurement_table_name(datum.datum_type()); + let row = serde_json::to_string(&inserted_row).unwrap(); + let insert_sql = format!( + "INSERT INTO oximeter.{measurement_table} FORMAT JSONEachRow {row}", + ); + client + .execute(insert_sql) + .await + .expect("Failed to insert measurement row"); + + // Select it exactly back out. + let select_sql = format!( + "SELECT * FROM oximeter.{} LIMIT 1 FORMAT {};", + measurement_table, + crate::DATABASE_SELECT_FORMAT, + ); + let body = client + .execute_with_body(select_sql) + .await + .expect("Failed to select measurement row"); + let actual_row: serde_json::Value = serde_json::from_str(&body) + .expect("Failed to parse measurement row JSON"); + println!("{actual_row:?}"); + println!("{inserted_row:?}"); + assert_eq!( + actual_row, inserted_row, + "Actual and expected measurement rows do not match" + ); + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) + } + + // Returns the number of timeseries schemas being used. + async fn get_schema_count(client: &Client) -> usize { + client + .execute_with_body( + "SELECT * FROM oximeter.timeseries_schema FORMAT JSONEachRow;", + ) + .await + .expect("Failed to SELECT from database") + .lines() + .count() + } + + async fn database_version_update_idempotent_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_database_version_update_idempotent"); let log = &logctx.log; + + let replicated = false; + + // Initialize the database... let client = Client::new(address, &log); client - .init_single_node_db() + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) .await .expect("Failed to initialize timeseries database"); + + // Insert data here so we can verify it still exists later. + // + // The values here don't matter much, we just want to check that + // the database data hasn't been dropped. + assert_eq!(0, get_schema_count(&client).await); + let sample = test_util::make_sample(); + client.insert_samples(&[sample.clone()]).await.unwrap(); + assert_eq!(1, get_schema_count(&client).await); + + // Re-initialize the database, see that our data still exists client - .insert_samples(&samples) - .await - .expect("Failed to insert samples"); - let timeseries_name = "service:request_latency"; - let start_time = samples[samples.len() / 2].measurement.timestamp(); - let mut timeseries = client - .select_timeseries_with( - timeseries_name, - &[], - Some(query::Timestamp::Exclusive(start_time)), - None, - None, - None, - ) + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) .await - .expect("Failed to select timeseries"); - timeseries.sort_by(|first, second| { - first.measurements[0] - .timestamp() - .cmp(&second.measurements[0].timestamp()) - }); - assert_eq!(timeseries.len(), metrics.len() / 2); - for ts in timeseries.iter() { - for meas in ts.measurements.iter() { - assert!(meas.timestamp() > start_time); - } - } - db.cleanup().await.expect("Failed to cleanup database"); + .expect("Failed to initialize timeseries database"); + + assert_eq!(1, get_schema_count(&client).await); + + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_select_timeseries_with_limit() { - let (_, _, samples) = setup_select_test(); - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let logctx = test_setup_log("test_select_timeseries_with_limit"); + async fn database_version_will_not_downgrade_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_database_version_will_not_downgrade"); let log = &logctx.log; + + let replicated = false; + + // Initialize the database let client = Client::new(address, &log); client - .init_single_node_db() + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) .await .expect("Failed to initialize timeseries database"); + + // Bump the version of the database to a "too new" version client - .insert_samples(&samples) + .insert_version(model::OXIMETER_VERSION + 1) .await - .expect("Failed to insert samples"); - let timeseries_name = "service:request_latency"; - - // So we have to define criteria that resolve to a single timeseries - let criteria = - &["name==oximeter", "route==/a", "method==GET", "status_code==200"]; + .expect("Failed to insert very new DB version"); - // First, query without a limit. We should see all the results. - let all_measurements = &client - .select_timeseries_with( - timeseries_name, - criteria, - None, - None, - None, - None, - ) + // Expect a failure re-initializing the client. + // + // This will attempt to initialize the client with "version = + // model::OXIMETER_VERSION", which is "too old". + client + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) .await - .expect("Failed to select timeseries")[0] - .measurements; + .expect_err("Should have failed, downgrades are not supported"); - // Check some constraints on the number of measurements - we - // can change these, but these assumptions make the test simpler. - // - // For now, assume we can cleanly cut the number of measurements in - // half. - assert!(all_measurements.len() >= 2); - assert!(all_measurements.len() % 2 == 0); + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) + } - // Next, let's set a limit to half the results and query again. - let limit = - NonZeroU32::new(u32::try_from(all_measurements.len() / 2).unwrap()) - .unwrap(); + async fn database_version_wipes_old_version_test( + address: SocketAddr, + ) -> Result<(), Error> { + let logctx = test_setup_log("test_database_version_wipes_old_version"); + let log = &logctx.log; - // We expect queries with a limit to fail when they fail to resolve to a - // single timeseries, so run a query without any criteria to test that + let replicated = false; + + // Initialize the Client + let client = Client::new(address, &log); client - .select_timeseries_with( - timeseries_name, - &[], - None, - None, - Some(limit), - None, - ) + .initialize_db_with_version(replicated, model::OXIMETER_VERSION) .await - .expect_err("Should fail to select timeseries"); + .expect("Failed to initialize timeseries database"); - // queries with limit do not fail when they resolve to zero timeseries - let empty_result = &client - .select_timeseries_with( - timeseries_name, - &["name==not_a_real_name"], - None, - None, - Some(limit), - None, - ) - .await - .expect("Failed to select timeseries"); - assert_eq!(empty_result.len(), 0); + // Insert data here so we can remove it later. + // + // The values here don't matter much, we just want to check that + // the database data gets dropped later. + assert_eq!(0, get_schema_count(&client).await); + let sample = test_util::make_sample(); + client.insert_samples(&[sample.clone()]).await.unwrap(); + assert_eq!(1, get_schema_count(&client).await); - let timeseries = &client - .select_timeseries_with( - timeseries_name, - criteria, - None, - None, - Some(limit), - None, - ) + // If we try to upgrade to a newer version, we'll drop old data. + client + .initialize_db_with_version(replicated, model::OXIMETER_VERSION + 1) .await - .expect("Failed to select timeseries")[0]; - assert_eq!(timeseries.measurements.len() as u32, limit.get()); - assert_eq!( - all_measurements[..all_measurements.len() / 2], - timeseries.measurements - ); + .expect("Should have initialized database successfully"); + assert_eq!(0, get_schema_count(&client).await); - // Get the other half of the results. - let timeseries = &client - .select_timeseries_with( - timeseries_name, - criteria, - Some(query::Timestamp::Exclusive( - timeseries.measurements.last().unwrap().timestamp(), - )), - None, - Some(limit), - None, - ) + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) + } + + async fn update_schema_cache_on_new_sample_test( + address: SocketAddr, + ) -> Result<(), Error> { + usdt::register_probes().unwrap(); + let logctx = test_setup_log("test_update_schema_cache_on_new_sample"); + let log = &logctx.log; + + let client = Client::new(address, &log); + client + .init_single_node_db() .await - .expect("Failed to select timeseries")[0]; - assert_eq!(timeseries.measurements.len() as u32, limit.get()); + .expect("Failed to initialize timeseries database"); + let samples = [test_util::make_sample()]; + client.insert_samples(&samples).await.unwrap(); + + // Get the count of schema directly from the DB, which should have just + // one. + let response = client.execute_with_body( + "SELECT COUNT() FROM oximeter.timeseries_schema FORMAT JSONEachRow; + ").await.unwrap(); + assert_eq!(response.lines().count(), 1, "Expected exactly 1 schema"); + assert_eq!(client.schema.lock().await.len(), 1); + + // Clear the internal cache, and insert the sample again. + // + // This should cause us to look up the schema in the DB again, but _not_ + // insert a new one. + client.schema.lock().await.clear(); + assert!(client.schema.lock().await.is_empty()); + + client.insert_samples(&samples).await.unwrap(); + + // Get the count of schema directly from the DB, which should still have + // only the one schema. + let response = client.execute_with_body( + "SELECT COUNT() FROM oximeter.timeseries_schema FORMAT JSONEachRow; + ").await.unwrap(); assert_eq!( - all_measurements[all_measurements.len() / 2..], - timeseries.measurements + response.lines().count(), + 1, + "Expected exactly 1 schema again" ); - - db.cleanup().await.expect("Failed to cleanup database"); + assert_eq!(client.schema.lock().await.len(), 1); + client.wipe_single_node_db().await?; logctx.cleanup_successful(); + Ok(()) } - #[tokio::test] - async fn test_select_timeseries_with_order() { - let (_, _, samples) = setup_select_test(); - let mut db = ClickHouseInstance::new_single_node(0) - .await - .expect("Failed to start ClickHouse"); - let address = SocketAddr::new("::1".parse().unwrap(), db.port()); - let logctx = test_setup_log("test_select_timeseries_with_order"); + // Regression test for https://github.com/oxidecomputer/omicron/issues/4336. + // + // This tests that we can successfully query all extant datum types from the + // schema table. There may be no such values, but the query itself should + // succeed. + async fn select_all_datum_types_test( + address: SocketAddr, + ) -> Result<(), Error> { + use strum::IntoEnumIterator; + usdt::register_probes().unwrap(); + let logctx = test_setup_log("test_update_schema_cache_on_new_sample"); let log = &logctx.log; + let client = Client::new(address, &log); client .init_single_node_db() .await .expect("Failed to initialize timeseries database"); - client - .insert_samples(&samples) - .await - .expect("Failed to insert samples"); - let timeseries_name = "service:request_latency"; - // Limits only work with a single timeseries, so we have to use criteria - // that resolve to a single timeseries - let criteria = - &["name==oximeter", "route==/a", "method==GET", "status_code==200"]; + // Attempt to select all schema with each datum type. + for ty in oximeter::DatumType::iter() { + let sql = format!( + "SELECT COUNT() \ + FROM {}.timeseries_schema WHERE \ + datum_type = '{:?}'", + crate::DATABASE_NAME, + crate::model::DbDatumType::from(ty), + ); + let res = client.execute_with_body(sql).await.unwrap(); + let count = res.trim().parse::().unwrap(); + assert_eq!(count, 0); + } + client.wipe_single_node_db().await?; + logctx.cleanup_successful(); + Ok(()) + } - // First, query without an order. We should see all the results in ascending order. - let all_measurements = &client - .select_timeseries_with( - timeseries_name, - criteria, - None, - None, - None, - None, - ) + // Regression test for https://github.com/oxidecomputer/omicron/issues/4335. + // + // This tests that, when cache new schema but _fail_ to insert them, we also + // remove them from the internal cache. + async fn new_schema_removed_when_not_inserted_test( + address: SocketAddr, + ) -> Result<(), Error> { + usdt::register_probes().unwrap(); + let logctx = test_setup_log("test_update_schema_cache_on_new_sample"); + let log = &logctx.log; + + let client = Client::new(address, &log); + client + .init_single_node_db() .await - .expect("Failed to select timeseries")[0] - .measurements; + .expect("Failed to initialize timeseries database"); + let samples = [test_util::make_sample()]; + // We're using the components of the `insert_samples()` method here, + // which has been refactored explicitly for this test. We need to insert + // the schema for this sample into the internal cache, which relies on + // access to the database (since they don't exist). + // + // First, insert the sample into the local cache. This method also + // checks the DB, since this schema doesn't exist in the cache. + let UnrolledSampleRows { new_schema, .. } = + client.unroll_samples(&samples).await; + assert_eq!(client.schema.lock().await.len(), 1); + + // Next, we'll kill the database, and then try to insert the schema. + // That will fail, since the DB is now inaccessible. + client.wipe_single_node_db().await?; + let res = client.save_new_schema_or_remove(new_schema).await; + assert!(res.is_err(), "Should have failed since the DB is gone"); assert!( - all_measurements.len() > 1, - "need more than one measurement to test ordering" + client.schema.lock().await.is_empty(), + "Failed to remove new schema from the cache when \ + they could not be inserted into the DB" ); + logctx.cleanup_successful(); + Ok(()) + } + + #[tokio::test] + async fn test_build_replicated() { + let logctx = test_setup_log("test_build_replicated"); + let log = &logctx.log; - // Explicitly specifying asc should give the same results - let timeseries_asc = &client - .select_timeseries_with( - timeseries_name, - criteria, - None, - None, - None, - Some(PaginationOrder::Ascending), - ) + let mut cluster = ClickHouseCluster::new() .await - .expect("Failed to select timeseries")[0] - .measurements; - assert_eq!(all_measurements, timeseries_asc); + .expect("Failed to initialise ClickHouse Cluster"); - // Now get the results in reverse order - let timeseries_desc = &client - .select_timeseries_with( - timeseries_name, - criteria, - None, - None, - None, - Some(PaginationOrder::Descending), - ) + // Create database in node 1 + let client_1 = Client::new(cluster.replica_1.address.unwrap(), &log); + assert!(client_1.is_oximeter_cluster().await.unwrap()); + client_1 + .init_replicated_db() .await - .expect("Failed to select timeseries")[0] - .measurements; + .expect("Failed to initialize timeseries database"); - let mut timeseries_asc_rev = timeseries_asc.clone(); - timeseries_asc_rev.reverse(); + // Verify database exists in node 2 + let client_2 = Client::new(cluster.replica_2.address.unwrap(), &log); + assert!(client_2.is_oximeter_cluster().await.unwrap()); + let sql = String::from("SHOW DATABASES FORMAT JSONEachRow;"); + let result = client_2.execute_with_body(sql).await.unwrap(); - assert_ne!(timeseries_desc, timeseries_asc); - assert_eq!(timeseries_desc, ×eries_asc_rev); + // Try a few times to make sure data has been synchronised. + let tries = 5; + for _ in 0..tries { + if !result.contains("oximeter") { + sleep(Duration::from_secs(1)).await; + continue; + } else { + break; + } + } - // can use limit 1 to get single most recent measurement - let desc_limit_1 = &client - .select_timeseries_with( - timeseries_name, - criteria, - None, - None, - Some(NonZeroU32::new(1).unwrap()), - Some(PaginationOrder::Descending), - ) - .await - .expect("Failed to select timeseries")[0] - .measurements; + assert!(result.contains("oximeter")); - assert_eq!(desc_limit_1.len(), 1); - assert_eq!( - desc_limit_1.first().unwrap(), - timeseries_asc.last().unwrap() + // Insert row into one of the tables + let sql = String::from( + "INSERT INTO oximeter.measurements_string (datum) VALUES ('hiya');", ); + client_2.execute_with_body(sql).await.unwrap(); + + let sql = String::from( + "SELECT * FROM oximeter.measurements_string FORMAT JSONEachRow;", + ); + let result = client_2.execute_with_body(sql.clone()).await.unwrap(); + assert!(result.contains("hiya")); + + // TODO(https://github.com/oxidecomputer/omicron/issues/4001): With distributed + // engine, it can take a long time to sync the data. This means it's tricky to + // test that the data exists on both nodes. + + cluster + .keeper_1 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse keeper 1"); + cluster + .keeper_2 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse keeper 2"); + cluster + .keeper_3 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse keeper 3"); + cluster + .replica_1 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse server 1"); + cluster + .replica_2 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse server 2"); - db.cleanup().await.expect("Failed to cleanup database"); logctx.cleanup_successful(); } - #[tokio::test] - async fn test_get_schema_no_new_values() { - let (mut db, client, _) = setup_filter_testcase().await; - let schema = &client.schema.lock().unwrap().clone(); - client.get_schema().await.expect("Failed to get timeseries schema"); - assert_eq!( - schema, - &*client.schema.lock().unwrap(), - "Schema shouldn't change" - ); - db.cleanup().await.expect("Failed to cleanup database"); + // Testing helper functions + + #[derive(Debug, Clone, oximeter::Target)] + struct Service { + name: String, + id: uuid::Uuid, + } + #[derive(Debug, Clone, oximeter::Metric)] + struct RequestLatency { + route: String, + method: String, + status_code: i64, + #[datum] + latency: f64, } - #[tokio::test] - async fn test_timeseries_schema_list() { - use std::convert::TryInto; + const SELECT_TEST_ID: &str = "4fa827ea-38bb-c37e-ac2d-f8432ca9c76e"; + fn setup_select_test() -> (Service, Vec, Vec) { + // One target + let id = SELECT_TEST_ID.parse().unwrap(); + let target = Service { name: "oximeter".to_string(), id }; - let (mut db, client, _) = setup_filter_testcase().await; - let limit = 100u32.try_into().unwrap(); - let page = dropshot::WhichPage::First(dropshot::EmptyScanParams {}); - let result = client.timeseries_schema_list(&page, limit).await.unwrap(); - assert!( - result.items.len() == 1, - "Expected exactly 1 timeseries schema" - ); - let last_seen = result.items.last().unwrap().timeseries_name.clone(); - let page = dropshot::WhichPage::Next(last_seen); - let result = client.timeseries_schema_list(&page, limit).await.unwrap(); - assert!( - result.items.is_empty(), - "Expected the next page of schema to be empty" - ); - assert!( - result.next_page.is_none(), - "Expected the next page token to be None" - ); - db.cleanup().await.expect("Failed to cleanup database"); + // Many metrics + let routes = &["/a", "/b"]; + let methods = &["GET", "POST"]; + let status_codes = &[200, 204, 500]; + + // Two samples each + let n_timeseries = routes.len() * methods.len() * status_codes.len(); + let mut metrics = Vec::with_capacity(n_timeseries); + let mut samples = Vec::with_capacity(n_timeseries * 2); + for (route, method, status_code) in + itertools::iproduct!(routes, methods, status_codes) + { + let metric = RequestLatency { + route: route.to_string(), + method: method.to_string(), + status_code: *status_code, + latency: 0.0, + }; + samples.push(Sample::new(&target, &metric).unwrap()); + samples.push(Sample::new(&target, &metric).unwrap()); + metrics.push(metric); + } + (target, metrics, samples) } - #[tokio::test] - async fn test_list_timeseries() { - use std::convert::TryInto; + // Small helper to go from a multidimensional index to a flattened array index. + fn unravel_index(idx: &[usize; 4]) -> usize { + let strides = [12, 6, 2, 1]; + let mut index = 0; + for (i, stride) in idx.iter().rev().zip(strides.iter().rev()) { + index += i * stride; + } + index + } - let (mut db, client, _) = setup_filter_testcase().await; - let limit = 7u32.try_into().unwrap(); - let params = crate::TimeseriesScanParams { - timeseries_name: TimeseriesName::try_from( - "virtual_machine:cpu_busy", - ) - .unwrap(), - criteria: vec!["cpu_id==0".parse().unwrap()], - start_time: None, - end_time: None, - }; - let page = dropshot::WhichPage::First(params.clone()); - let result = client.list_timeseries(&page, limit).await.unwrap(); + #[test] + fn test_unravel_index() { + assert_eq!(unravel_index(&[0, 0, 0, 0]), 0); + assert_eq!(unravel_index(&[0, 0, 1, 0]), 2); + assert_eq!(unravel_index(&[1, 0, 0, 0]), 12); + assert_eq!(unravel_index(&[1, 0, 0, 1]), 13); + assert_eq!(unravel_index(&[1, 0, 1, 0]), 14); + assert_eq!(unravel_index(&[1, 1, 2, 1]), 23); + } - // We should have 4 timeseries, with 2 samples from each of the first 3 and 1 from the last - // timeseries. - assert_eq!(result.items.len(), 4, "Expected 4 timeseries"); - assert_eq!( - result.items.last().unwrap().measurements.len(), - 1, - "Expected 1 sample from the last timeseries" - ); - assert!( - result.items.iter().take(3).all(|ts| ts.measurements.len() == 2), - "Expected 2 samples from the first 3 timeseries" - ); - let last_timeseries = result.items.last().unwrap(); + fn verify_measurements( + measurements: &[oximeter::Measurement], + samples: &[Sample], + ) { + for (measurement, sample) in measurements.iter().zip(samples.iter()) { + assert_eq!( + measurement, &sample.measurement, + "Mismatch between retrieved and expected measurement", + ); + } + } - // Get the next page. - // - // We have to recreate this as dropshot would, since we cannot build the pagination params - // ourselves. - let next_page = crate::TimeseriesPageSelector { params, offset: limit }; - let page = dropshot::WhichPage::Next(next_page); - let result = client.list_timeseries(&page, limit).await.unwrap(); + fn verify_target(actual: &crate::Target, expected: &Service) { + assert_eq!(actual.name, expected.name()); + for (field_name, field_value) in expected + .field_names() + .into_iter() + .zip(expected.field_values().into_iter()) + { + let actual_field = actual + .fields + .iter() + .find(|f| f.name == *field_name) + .expect("Missing field in recovered timeseries target"); + assert_eq!( + actual_field.value, field_value, + "Incorrect field value in timeseries target" + ); + } + } - // We should now have the one remaining sample - assert_eq!( - result.items.len(), - 1, - "Expected only 1 timeseries after paginating" - ); - assert_eq!( - result.items[0].measurements.len(), - 1, - "Expected only the last sample after paginating" - ); - assert_eq!( - result.items[0].timeseries_name, last_timeseries.timeseries_name, - "Paginating should pick up where it left off" - ); - assert_eq!( - result.items[0].target, last_timeseries.target, - "Paginating should pick up where it left off" - ); - assert_eq!( - result.items[0].metric, last_timeseries.metric, - "Paginating should pick up where it left off" - ); - db.cleanup().await.expect("Failed to cleanup database"); + fn verify_metric(actual: &crate::Metric, expected: &RequestLatency) { + assert_eq!(actual.name, expected.name()); + for (field_name, field_value) in expected + .field_names() + .into_iter() + .zip(expected.field_values().into_iter()) + { + let actual_field = actual + .fields + .iter() + .find(|f| f.name == *field_name) + .expect("Missing field in recovered timeseries metric"); + assert_eq!( + actual_field.value, field_value, + "Incorrect field value in timeseries metric" + ); + } } } diff --git a/oximeter/db/src/configs/replica_config.xml b/oximeter/db/src/configs/replica_config.xml index 6a2cab5862..bf424185b6 100644 --- a/oximeter/db/src/configs/replica_config.xml +++ b/oximeter/db/src/configs/replica_config.xml @@ -321,11 +321,11 @@ true - 9000 + 9001 - 9001 + 9002 diff --git a/oximeter/db/src/db-replicated-init.sql b/oximeter/db/src/db-replicated-init.sql index 7b756f4b0d..21a647b1a5 100644 --- a/oximeter/db/src/db-replicated-init.sql +++ b/oximeter/db/src/db-replicated-init.sql @@ -652,7 +652,25 @@ CREATE TABLE IF NOT EXISTS oximeter.timeseries_schema ON CLUSTER oximeter_cluste 'CumulativeI64' = 6, 'CumulativeF64' = 7, 'HistogramI64' = 8, - 'HistogramF64' = 9 + 'HistogramF64' = 9, + 'I8' = 10, + 'U8' = 11, + 'I16' = 12, + 'U16' = 13, + 'I32' = 14, + 'U32' = 15, + 'U64' = 16, + 'F32' = 17, + 'CumulativeU64' = 18, + 'CumulativeF32' = 19, + 'HistogramI8' = 20, + 'HistogramU8' = 21, + 'HistogramI16' = 22, + 'HistogramU16' = 23, + 'HistogramI32' = 24, + 'HistogramU32' = 25, + 'HistogramU64' = 26, + 'HistogramF32' = 27 ), created DateTime64(9, 'UTC') ) diff --git a/oximeter/db/src/db-single-node-init.sql b/oximeter/db/src/db-single-node-init.sql index 1f648fd5d5..2fb5c36397 100644 --- a/oximeter/db/src/db-single-node-init.sql +++ b/oximeter/db/src/db-single-node-init.sql @@ -476,7 +476,25 @@ CREATE TABLE IF NOT EXISTS oximeter.timeseries_schema 'CumulativeI64' = 6, 'CumulativeF64' = 7, 'HistogramI64' = 8, - 'HistogramF64' = 9 + 'HistogramF64' = 9, + 'I8' = 10, + 'U8' = 11, + 'I16' = 12, + 'U16' = 13, + 'I32' = 14, + 'U32' = 15, + 'U64' = 16, + 'F32' = 17, + 'CumulativeU64' = 18, + 'CumulativeF32' = 19, + 'HistogramI8' = 20, + 'HistogramU8' = 21, + 'HistogramI16' = 22, + 'HistogramU16' = 23, + 'HistogramI32' = 24, + 'HistogramU32' = 25, + 'HistogramU64' = 26, + 'HistogramF32' = 27 ), created DateTime64(9, 'UTC') ) diff --git a/oximeter/db/src/lib.rs b/oximeter/db/src/lib.rs index c878b8ff2a..11ecbeddc8 100644 --- a/oximeter/db/src/lib.rs +++ b/oximeter/db/src/lib.rs @@ -4,7 +4,7 @@ //! Tools for interacting with the control plane telemetry database. -// Copyright 2021 Oxide Computer Company +// Copyright 2023 Oxide Computer Company use crate::query::StringFieldSelector; use chrono::{DateTime, Utc}; @@ -13,6 +13,7 @@ pub use oximeter::{DatumType, Field, FieldType, Measurement, Sample}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::convert::TryFrom; use std::num::NonZeroU32; use thiserror::Error; @@ -36,12 +37,8 @@ pub enum Error { Database(String), /// A schema provided when collecting samples did not match the expected schema - #[error("Schema mismatch for timeseries '{name}', expected fields {expected:?} found fields {actual:?}")] - SchemaMismatch { - name: String, - expected: BTreeMap, - actual: BTreeMap, - }, + #[error("Schema mismatch for timeseries '{0}'", expected.timeseries_name)] + SchemaMismatch { expected: TimeseriesSchema, actual: TimeseriesSchema }, #[error("Timeseries not found for: {0}")] TimeseriesNotFound(String), @@ -153,6 +150,13 @@ impl std::convert::TryFrom for TimeseriesName { } } +impl std::str::FromStr for TimeseriesName { + type Err = Error; + fn from_str(s: &str) -> Result { + s.try_into() + } +} + impl PartialEq for TimeseriesName where T: AsRef, @@ -177,7 +181,7 @@ fn validate_timeseries_name(s: &str) -> Result<&str, Error> { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct TimeseriesSchema { pub timeseries_name: TimeseriesName, - pub field_schema: Vec, + pub field_schema: BTreeSet, pub datum_type: DatumType, pub created: DateTime, } @@ -398,6 +402,8 @@ const TIMESERIES_NAME_REGEX: &str = #[cfg(test)] mod tests { use super::*; + use crate::model::DbFieldList; + use crate::model::DbTimeseriesSchema; use std::convert::TryFrom; use uuid::Uuid; @@ -505,4 +511,122 @@ mod tests { &output.join("\n"), ); } + + // Test that we correctly order field across a target and metric. + // + // In an earlier commit, we switched from storing fields in an unordered Vec + // to using a BTree{Map,Set} to ensure ordering by name. However, the + // `TimeseriesSchema` type stored all its fields by chaining the sorted + // fields from the target and metric, without then sorting _across_ them. + // + // This was exacerbated by the error reporting, where we did in fact sort + // all fields across the target and metric, making it difficult to tell how + // the derived schema was different, if at all. + // + // This test generates a sample with a schema where the target and metric + // fields are sorted within them, but not across them. We check that the + // derived schema are actually equal, which means we've imposed that + // ordering when deriving the schema. + #[test] + fn test_schema_field_ordering_across_target_metric() { + let target_field = FieldSchema { + name: String::from("later"), + ty: FieldType::U64, + source: FieldSource::Target, + }; + let metric_field = FieldSchema { + name: String::from("earlier"), + ty: FieldType::U64, + source: FieldSource::Metric, + }; + let timeseries_name: TimeseriesName = "foo:bar".parse().unwrap(); + let datum_type = DatumType::U64; + let field_schema = + [target_field.clone(), metric_field.clone()].into_iter().collect(); + let expected_schema = TimeseriesSchema { + timeseries_name, + field_schema, + datum_type, + created: Utc::now(), + }; + + #[derive(oximeter::Target)] + struct Foo { + later: u64, + } + #[derive(oximeter::Metric)] + struct Bar { + earlier: u64, + datum: u64, + } + + let target = Foo { later: 1 }; + let metric = Bar { earlier: 2, datum: 10 }; + let sample = Sample::new(&target, &metric).unwrap(); + let derived_schema = model::schema_for(&sample); + assert_eq!(derived_schema, expected_schema); + } + + #[test] + fn test_unsorted_db_fields_are_sorted_on_read() { + let target_field = FieldSchema { + name: String::from("later"), + ty: FieldType::U64, + source: FieldSource::Target, + }; + let metric_field = FieldSchema { + name: String::from("earlier"), + ty: FieldType::U64, + source: FieldSource::Metric, + }; + let timeseries_name: TimeseriesName = "foo:bar".parse().unwrap(); + let datum_type = DatumType::U64; + let field_schema = + [target_field.clone(), metric_field.clone()].into_iter().collect(); + let expected_schema = TimeseriesSchema { + timeseries_name: timeseries_name.clone(), + field_schema, + datum_type, + created: Utc::now(), + }; + + // The fields here are sorted by target and then metric, which is how we + // used to insert them into the DB. We're checking that they are totally + // sorted when we read them out of the DB, even though they are not in + // the extracted model type. + let db_fields = DbFieldList { + names: vec![target_field.name.clone(), metric_field.name.clone()], + types: vec![target_field.ty.into(), metric_field.ty.into()], + sources: vec![ + target_field.source.into(), + metric_field.source.into(), + ], + }; + let db_schema = DbTimeseriesSchema { + timeseries_name: timeseries_name.to_string(), + field_schema: db_fields, + datum_type: datum_type.into(), + created: expected_schema.created, + }; + assert_eq!(expected_schema, TimeseriesSchema::from(db_schema)); + } + + #[test] + fn test_field_schema_ordering() { + let mut fields = BTreeSet::new(); + fields.insert(FieldSchema { + name: String::from("second"), + ty: FieldType::U64, + source: FieldSource::Target, + }); + fields.insert(FieldSchema { + name: String::from("first"), + ty: FieldType::U64, + source: FieldSource::Target, + }); + let mut iter = fields.iter(); + assert_eq!(iter.next().unwrap().name, "first"); + assert_eq!(iter.next().unwrap().name, "second"); + assert!(iter.next().is_none()); + } } diff --git a/oximeter/db/src/model.rs b/oximeter/db/src/model.rs index 1314c5c649..41c7ab9d24 100644 --- a/oximeter/db/src/model.rs +++ b/oximeter/db/src/model.rs @@ -30,6 +30,7 @@ use oximeter::types::Sample; use serde::Deserialize; use serde::Serialize; use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::convert::TryFrom; use std::net::IpAddr; use std::net::Ipv6Addr; @@ -41,7 +42,7 @@ use uuid::Uuid; /// /// TODO(#4271): The current implementation of versioning will wipe the metrics /// database if this number is incremented. -pub const OXIMETER_VERSION: u64 = 1; +pub const OXIMETER_VERSION: u64 = 2; // Wrapper type to represent a boolean in the database. // @@ -107,7 +108,7 @@ pub(crate) struct DbFieldList { pub sources: Vec, } -impl From for Vec { +impl From for BTreeSet { fn from(list: DbFieldList) -> Self { list.names .into_iter() @@ -122,8 +123,8 @@ impl From for Vec { } } -impl From> for DbFieldList { - fn from(list: Vec) -> Self { +impl From> for DbFieldList { + fn from(list: BTreeSet) -> Self { let mut names = Vec::with_capacity(list.len()); let mut types = Vec::with_capacity(list.len()); let mut sources = Vec::with_capacity(list.len()); @@ -914,6 +915,9 @@ pub(crate) fn unroll_measurement_row(sample: &Sample) -> (String, String) { /// Return the schema for a `Sample`. pub(crate) fn schema_for(sample: &Sample) -> TimeseriesSchema { + // The fields are iterated through whatever order the `Target` or `Metric` + // impl chooses. We'll store in a set ordered by field name, to ignore the + // declaration order. let created = Utc::now(); let field_schema = sample .target_fields() @@ -1403,7 +1407,7 @@ mod tests { sources: vec![DbFieldSource::Target, DbFieldSource::Metric], }; - let list = vec![ + let list: BTreeSet<_> = [ FieldSchema { name: String::from("field0"), ty: FieldType::I64, @@ -1414,11 +1418,13 @@ mod tests { ty: FieldType::IpAddr, source: FieldSource::Metric, }, - ]; + ] + .into_iter() + .collect(); assert_eq!(DbFieldList::from(list.clone()), db_list); assert_eq!(db_list, list.clone().into()); - let round_trip: Vec = + let round_trip: BTreeSet = DbFieldList::from(list.clone()).into(); assert_eq!(round_trip, list); } diff --git a/oximeter/db/src/query.rs b/oximeter/db/src/query.rs index e9e1600739..6a55d3f518 100644 --- a/oximeter/db/src/query.rs +++ b/oximeter/db/src/query.rs @@ -721,6 +721,7 @@ mod tests { use crate::FieldSource; use crate::TimeseriesName; use chrono::NaiveDateTime; + use std::collections::BTreeSet; use std::convert::TryFrom; #[test] @@ -774,7 +775,7 @@ mod tests { fn test_select_query_builder_filter_raw() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), - field_schema: vec![ + field_schema: [ FieldSchema { name: "f0".to_string(), ty: FieldType::I64, @@ -785,7 +786,9 @@ mod tests { ty: FieldType::Bool, source: FieldSource::Target, }, - ], + ] + .into_iter() + .collect(), datum_type: DatumType::I64, created: Utc::now(), }; @@ -905,7 +908,7 @@ mod tests { fn test_select_query_builder_no_fields() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), - field_schema: vec![], + field_schema: BTreeSet::new(), datum_type: DatumType::I64, created: Utc::now(), }; @@ -927,7 +930,7 @@ mod tests { fn test_select_query_builder_limit_offset() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), - field_schema: vec![], + field_schema: BTreeSet::new(), datum_type: DatumType::I64, created: Utc::now(), }; @@ -996,7 +999,7 @@ mod tests { fn test_select_query_builder_no_selectors() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), - field_schema: vec![ + field_schema: [ FieldSchema { name: "f0".to_string(), ty: FieldType::I64, @@ -1007,7 +1010,9 @@ mod tests { ty: FieldType::Bool, source: FieldSource::Target, }, - ], + ] + .into_iter() + .collect(), datum_type: DatumType::I64, created: Utc::now(), }; @@ -1057,7 +1062,7 @@ mod tests { fn test_select_query_builder_field_selectors() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), - field_schema: vec![ + field_schema: [ FieldSchema { name: "f0".to_string(), ty: FieldType::I64, @@ -1068,7 +1073,9 @@ mod tests { ty: FieldType::Bool, source: FieldSource::Target, }, - ], + ] + .into_iter() + .collect(), datum_type: DatumType::I64, created: Utc::now(), }; @@ -1106,7 +1113,7 @@ mod tests { fn test_select_query_builder_full() { let schema = TimeseriesSchema { timeseries_name: TimeseriesName::try_from("foo:bar").unwrap(), - field_schema: vec![ + field_schema: [ FieldSchema { name: "f0".to_string(), ty: FieldType::I64, @@ -1117,7 +1124,9 @@ mod tests { ty: FieldType::Bool, source: FieldSource::Target, }, - ], + ] + .into_iter() + .collect(), datum_type: DatumType::I64, created: Utc::now(), }; diff --git a/oximeter/oximeter/src/test_util.rs b/oximeter/oximeter/src/test_util.rs index f3aae58fc8..f3750d6d83 100644 --- a/oximeter/oximeter/src/test_util.rs +++ b/oximeter/oximeter/src/test_util.rs @@ -104,3 +104,19 @@ pub fn generate_test_samples( } samples } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gen_test_samples() { + let (n_projects, n_instances, n_cpus, n_samples) = (2, 2, 2, 2); + let samples = + generate_test_samples(n_projects, n_instances, n_cpus, n_samples); + assert_eq!( + samples.len(), + n_projects * n_instances * n_cpus * n_samples + ); + } +} diff --git a/oximeter/oximeter/src/types.rs b/oximeter/oximeter/src/types.rs index d3f1b9e746..0cc3299ec4 100644 --- a/oximeter/oximeter/src/types.rs +++ b/oximeter/oximeter/src/types.rs @@ -275,6 +275,7 @@ pub struct Field { JsonSchema, Serialize, Deserialize, + strum::EnumIter, )] #[serde(rename_all = "snake_case")] pub enum DatumType { diff --git a/package-manifest.toml b/package-manifest.toml index a88f8170d0..b8ffb2756a 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -381,10 +381,10 @@ only_for_targets.image = "standard" # 3. Use source.type = "manual" instead of "prebuilt" source.type = "prebuilt" source.repo = "crucible" -source.commit = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source.commit = "da534e73380f3cc53ca0de073e1ea862ae32109b" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible.sha256.txt -source.sha256 = "010281ff5c3a0807c9e770d79264c954816a055aa482988d81e85ed98242e454" +source.sha256 = "572ac3b19e51b4e476266a62c2b7e06eff81c386cb48247c4b9f9b1e2ee81895" output.type = "zone" [package.crucible-pantry] @@ -392,10 +392,10 @@ service_name = "crucible_pantry" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "657d985247b41e38aac2e271c7ce8bc9ea81f4b6" +source.commit = "da534e73380f3cc53ca0de073e1ea862ae32109b" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-pantry.sha256.txt -source.sha256 = "809936edff2957e761e49667d5477e34b7a862050b4e082a59fdc95096d3bdd5" +source.sha256 = "812269958e18f54d72bc10bb4fb81f26c084cf762da7fd98e63d58c689be9ad1" output.type = "zone" # Refer to @@ -406,13 +406,13 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "334df299a56cd0d33e1227ed4ce4d2fe7478d541" +source.commit = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "531e0654de94b6e805836c35aa88b8a1ac691184000a03976e2b7825061e904e" +source.sha256 = "aa1d9dc5c9117c100f9636901e8eec6679d7dfbf869c46b7f2873585f94a1b89" output.type = "zone" -[package.maghemite] +[package.mg-ddm-gz] service_name = "mg-ddm" # Note: unlike every other package, `maghemite` is not restricted to either the # "standard" or "trampoline" image; it is included in both. @@ -422,10 +422,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "12703675393459e74139f8140e0b3c4c4f129d5d" +source.commit = "d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt -source.sha256 = "e57fe791ee898d59890c5779fbd4dce598250fb6ed53832024212bcdeec0cc5b" +source.sha256 = "d871406ed926571efebdab248de08d4f1ca6c31d4f9a691ce47b186474165c57" output.type = "tarball" [package.mg-ddm] @@ -438,10 +438,25 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "12703675393459e74139f8140e0b3c4c4f129d5d" +source.commit = "d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "3aa0d32b1d2b6be7091b9c665657296e924a86a00ca38756e9f45a1e629fd92b" +source.sha256 = "85ec05a8726989b5cb0a567de6b0855f6f84b6f3409ac99ccaf372be5821e45d" +output.type = "zone" +output.intermediate_only = true + +[package.mgd] +service_name = "mgd" +source.type = "prebuilt" +source.repo = "maghemite" +# Updating the commit hash here currently requires also updating +# `tools/maghemite_openapi_version`. Failing to do so will cause a failure when +# building `ddm-admin-client` (which will instruct you to update +# `tools/maghemite_openapi_version`). +source.commit = "d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" +# The SHA256 digest is automatically posted to: +# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt +source.sha256 = "452dfb3491e1b6d4df6be1cb689921f59623aed082e47606a78c0f44d918f66a" output.type = "zone" output.intermediate_only = true @@ -458,8 +473,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "7712104585266a2898da38c1345210ad26f9e71d" -source.sha256 = "486b0b016c0df06947810b90f3a3dd40423f0ee6f255ed079dc8e5618c9a7281" +source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966" +source.sha256 = "0808f331741e02d55e199847579dfd01f3658b21c7122cef8c3f9279f43dbab0" output.type = "zone" output.intermediate_only = true @@ -483,8 +498,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "7712104585266a2898da38c1345210ad26f9e71d" -source.sha256 = "76ff76d3526323c3fcbe2351cf9fbda4840e0dc11cd0eb6b71a3e0bd36c5e5e8" +source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966" +source.sha256 = "c359de1be5073a484d86d4c58e8656a36002ce1dc38506f97b730e21615ccae1" output.type = "zone" output.intermediate_only = true @@ -501,8 +516,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "7712104585266a2898da38c1345210ad26f9e71d" -source.sha256 = "b8e5c176070f9bc9ea0028de1999c77d66ea3438913664163975964effe4481b" +source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966" +source.sha256 = "110bfbfb2cf3d3471f3e3a64d26129c7a02f6c5857f9623ebb99690728c3b2ff" output.type = "zone" output.intermediate_only = true @@ -534,6 +549,7 @@ source.packages = [ "wicketd.tar.gz", "wicket.tar.gz", "mg-ddm.tar.gz", + "mgd.tar.gz", "switch_zone_setup.tar.gz", "xcvradm.tar.gz" ] @@ -555,6 +571,7 @@ source.packages = [ "wicketd.tar.gz", "wicket.tar.gz", "mg-ddm.tar.gz", + "mgd.tar.gz", "switch_zone_setup.tar.gz", "sp-sim-stub.tar.gz" ] @@ -576,6 +593,7 @@ source.packages = [ "wicketd.tar.gz", "wicket.tar.gz", "mg-ddm.tar.gz", + "mgd.tar.gz", "switch_zone_setup.tar.gz", "sp-sim-softnpu.tar.gz" ] diff --git a/schema/crdb/8.0.0/up01.sql b/schema/crdb/8.0.0/up01.sql new file mode 100644 index 0000000000..c617a0b634 --- /dev/null +++ b/schema/crdb/8.0.0/up01.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.service_kind ADD VALUE IF NOT EXISTS 'mgd'; diff --git a/schema/crdb/8.0.0/up02.sql b/schema/crdb/8.0.0/up02.sql new file mode 100644 index 0000000000..119e7b9a86 --- /dev/null +++ b/schema/crdb/8.0.0/up02.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.bgp_config ADD COLUMN IF NOT EXISTS bgp_announce_set_id UUID NOT NULL; diff --git a/schema/crdb/8.0.0/up03.sql b/schema/crdb/8.0.0/up03.sql new file mode 100644 index 0000000000..3705d4091e --- /dev/null +++ b/schema/crdb/8.0.0/up03.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config DROP COLUMN IF EXISTS bgp_announce_set_id; diff --git a/schema/crdb/8.0.0/up04.sql b/schema/crdb/8.0.0/up04.sql new file mode 100644 index 0000000000..c5a91796dd --- /dev/null +++ b/schema/crdb/8.0.0/up04.sql @@ -0,0 +1,5 @@ +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM ( + 'Firecode', + 'None', + 'Rs' +); diff --git a/schema/crdb/8.0.0/up05.sql b/schema/crdb/8.0.0/up05.sql new file mode 100644 index 0000000000..4d94bafb9f --- /dev/null +++ b/schema/crdb/8.0.0/up05.sql @@ -0,0 +1,11 @@ +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( + '0G', + '1G', + '10G', + '25G', + '40G', + '50G', + '100G', + '200G', + '400G' +); diff --git a/schema/crdb/8.0.0/up06.sql b/schema/crdb/8.0.0/up06.sql new file mode 100644 index 0000000000..e27800969c --- /dev/null +++ b/schema/crdb/8.0.0/up06.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; diff --git a/schema/crdb/8.0.0/up07.sql b/schema/crdb/8.0.0/up07.sql new file mode 100644 index 0000000000..c84ae8e5d2 --- /dev/null +++ b/schema/crdb/8.0.0/up07.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; diff --git a/schema/crdb/8.0.0/up08.sql b/schema/crdb/8.0.0/up08.sql new file mode 100644 index 0000000000..c84480feba --- /dev/null +++ b/schema/crdb/8.0.0/up08.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS hold_time INT8; diff --git a/schema/crdb/8.0.0/up09.sql b/schema/crdb/8.0.0/up09.sql new file mode 100644 index 0000000000..82f645c753 --- /dev/null +++ b/schema/crdb/8.0.0/up09.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS idle_hold_time INT8; diff --git a/schema/crdb/8.0.0/up10.sql b/schema/crdb/8.0.0/up10.sql new file mode 100644 index 0000000000..a672953991 --- /dev/null +++ b/schema/crdb/8.0.0/up10.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS delay_open INT8; diff --git a/schema/crdb/8.0.0/up11.sql b/schema/crdb/8.0.0/up11.sql new file mode 100644 index 0000000000..63f16a011f --- /dev/null +++ b/schema/crdb/8.0.0/up11.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS connect_retry INT8; diff --git a/schema/crdb/8.0.0/up12.sql b/schema/crdb/8.0.0/up12.sql new file mode 100644 index 0000000000..431d10cd3c --- /dev/null +++ b/schema/crdb/8.0.0/up12.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS keepalive INT8; diff --git a/schema/crdb/8.0.0/up13.sql b/schema/crdb/8.0.0/up13.sql new file mode 100644 index 0000000000..44bfd90b8c --- /dev/null +++ b/schema/crdb/8.0.0/up13.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.rack ADD COLUMN IF NOT EXISTS rack_subnet INET; diff --git a/schema/crdb/8.0.0/up14.sql b/schema/crdb/8.0.0/up14.sql new file mode 100644 index 0000000000..18ce39e61c --- /dev/null +++ b/schema/crdb/8.0.0/up14.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS omicron.public.bootstore_keys ( + key TEXT NOT NULL PRIMARY KEY, + generation INT8 NOT NULL +); diff --git a/schema/crdb/README.adoc b/schema/crdb/README.adoc index c15b51e374..f92748f101 100644 --- a/schema/crdb/README.adoc +++ b/schema/crdb/README.adoc @@ -73,38 +73,15 @@ SQL Validation, via Automated Tests: ==== Handling common schema changes -CockroachDB's schema includes a description of all of the database's CHECK -constraints. If a CHECK constraint is anonymous (i.e. it is written simply as -`CHECK ` and not `CONSTRAINT CHECK expression`), CRDB -assigns it a name based on the table and column to which the constraint applies. -The challenge is that CRDB identifies tables and columns using opaque -identifiers whose values depend on the order in which tables and views were -defined in the current database. This means that adding, removing, or renaming -objects needs to be done carefully to preserve the relative ordering of objects -in new databases created by `dbinit.sql` and upgraded databases created by -applying `up.sql` transformations. - -===== Adding new columns with constraints - -Strongly consider naming new constraints (`CONSTRAINT `) to -avoid the problems with anonymous constraints described above. - -===== Adding tables and views - -New tables and views must be added to the end of `dbinit.sql` so that the order -of preceding `CREATE` statements is left unchanged. If your changes fail the -`CHECK` constraints test and you get a constraint name diff like this... - -``` -NamedSqlValue { - column: "constraint_name", - value: Some( - String( -< "4101115737_149_10_not_null", -> "4101115737_148_10_not_null", -``` - -...then you've probably inadvertently added a table or view in the wrong place. +Although CockroachDB's schema includes some opaque internally-generated fields +that are order dependent - such as the names of anonymous CHECK constraints - +our schema comparison tools intentionally ignore these values. As a result, +when performing schema changes, the order of new tables and constraints should +generally not be important. + +As convention, however, we recommend keeping the `db_metadata` file at the end of +`dbinit.sql`, so that the database does not contain a version until it is fully +populated. ==== Adding new source tables to an existing view diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 07971c19ce..307a2888b7 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -63,7 +63,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.rack ( initialized BOOL NOT NULL, /* Used to configure the updates service URL */ - tuf_base_url STRING(512) + tuf_base_url STRING(512), + + /* The IPv6 underlay /56 prefix for the rack */ + rack_subnet INET ); /* @@ -198,7 +201,8 @@ CREATE TYPE IF NOT EXISTS omicron.public.service_kind AS ENUM ( 'nexus', 'ntp', 'oximeter', - 'tfport' + 'tfport', + 'mgd' ); CREATE TABLE IF NOT EXISTS omicron.public.service ( @@ -2441,10 +2445,14 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_route_config ( CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_bgp_peer_config ( port_settings_id UUID, - bgp_announce_set_id UUID NOT NULL, bgp_config_id UUID NOT NULL, interface_name TEXT, addr INET, + hold_time INT8, + idle_hold_time INT8, + delay_open INT8, + connect_retry INT8, + keepalive INT8, /* TODO https://github.com/oxidecomputer/omicron/issues/3013 */ PRIMARY KEY (port_settings_id, interface_name, addr) @@ -2458,7 +2466,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.bgp_config ( time_modified TIMESTAMPTZ NOT NULL, time_deleted TIMESTAMPTZ, asn INT8 NOT NULL, - vrf TEXT + vrf TEXT, + bgp_announce_set_id UUID NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS lookup_bgp_config_by_name ON omicron.public.bgp_config ( @@ -2500,6 +2509,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_address_config ( PRIMARY KEY (port_settings_id, address, interface_name) ); +CREATE TABLE IF NOT EXISTS omicron.public.bootstore_keys ( + key TEXT NOT NULL PRIMARY KEY, + generation INT8 NOT NULL +); + /* * The `sled_instance` view's definition needs to be modified in a separate * transaction from the transaction that created it. @@ -2730,37 +2744,6 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_caboose ( * nothing to ensure it gets bumped when it should be, but it's a start. */ -CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( - -- There should only be one row of this table for the whole DB. - -- It's a little goofy, but filter on "singleton = true" before querying - -- or applying updates, and you'll access the singleton row. - -- - -- We also add a constraint on this table to ensure it's not possible to - -- access the version of this table with "singleton = false". - singleton BOOL NOT NULL PRIMARY KEY, - time_created TIMESTAMPTZ NOT NULL, - time_modified TIMESTAMPTZ NOT NULL, - -- Semver representation of the DB version - version STRING(64) NOT NULL, - - -- (Optional) Semver representation of the DB version to which we're upgrading - target_version STRING(64), - - CHECK (singleton = true) -); - -INSERT INTO omicron.public.db_metadata ( - singleton, - time_created, - time_modified, - version, - target_version -) VALUES - ( TRUE, NOW(), NOW(), '7.0.0', NULL) -ON CONFLICT DO NOTHING; - - - -- Per-VMM state. CREATE TABLE IF NOT EXISTS omicron.public.vmm ( id UUID PRIMARY KEY, @@ -2807,4 +2790,54 @@ FROM WHERE instance.time_deleted IS NULL AND vmm.time_deleted IS NULL; +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM ( + 'Firecode', + 'None', + 'Rs' +); + +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( + '0G', + '1G', + '10G', + '25G', + '40G', + '50G', + '100G', + '200G', + '400G' +); + +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; + +CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( + -- There should only be one row of this table for the whole DB. + -- It's a little goofy, but filter on "singleton = true" before querying + -- or applying updates, and you'll access the singleton row. + -- + -- We also add a constraint on this table to ensure it's not possible to + -- access the version of this table with "singleton = false". + singleton BOOL NOT NULL PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + -- Semver representation of the DB version + version STRING(64) NOT NULL, + + -- (Optional) Semver representation of the DB version to which we're upgrading + target_version STRING(64), + + CHECK (singleton = true) +); + +INSERT INTO omicron.public.db_metadata ( + singleton, + time_created, + time_modified, + version, + target_version +) VALUES + ( TRUE, NOW(), NOW(), '8.0.0', NULL) +ON CONFLICT DO NOTHING; + COMMIT; diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 4a8b02d23d..39a9a68acc 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -91,6 +91,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "required": [ + "asn", + "originate" + ], + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/definitions/Ipv4Network" + } + } + } + }, + "BgpPeerConfig": { + "type": "object", + "required": [ + "addr", + "asn", + "port" + ], + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + } + }, "BootstrapAddressDiscovery": { "oneOf": [ { @@ -149,6 +196,27 @@ } } }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/definitions/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/definitions/Ipv6Network" + } + ] + } + ], + "x-rust-type": "ipnetwork::IpNetwork" + }, "IpRange": { "oneOf": [ { @@ -201,6 +269,11 @@ "type": "string", "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$", + "x-rust-type": "ipnetwork::Ipv6Network" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -244,6 +317,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ], + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/definitions/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/definitions/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/definitions/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/definitions/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/definitions/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/definitions/PortSpeed" + } + ] + } + } + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -336,7 +472,7 @@ "description": "Initial rack network configuration", "anyOf": [ { - "$ref": "#/definitions/RackNetworkConfig" + "$ref": "#/definitions/RackNetworkConfigV1" }, { "type": "null" @@ -367,15 +503,24 @@ } } }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ], "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/definitions/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -386,12 +531,15 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/definitions/UplinkConfig" + "$ref": "#/definitions/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/definitions/Ipv6Network" } } }, @@ -414,6 +562,28 @@ } } }, + "RouteConfig": { + "type": "object", + "required": [ + "destination", + "nexthop" + ], + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/definitions/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + } + }, "StartSledAgentRequest": { "description": "Configuration information for launching a Sled Agent.", "type": "object", @@ -484,69 +654,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ], - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/definitions/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/definitions/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/definitions/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/definitions/PortSpeed" - } - ] - }, - "uplink_vid": { - "description": "VLAN id to use for uplink", - "type": [ - "integer", - "null" - ], - "format": "uint16", - "minimum": 0.0 - } - } - }, "UserId": { "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", "type": "string" diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 82d7411d1a..ff9644773a 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -44,7 +44,6 @@ macaddr.workspace = true nexus-client.workspace = true omicron-common.workspace = true once_cell.workspace = true -oxide-vpc.workspace = true oximeter.workspace = true oximeter-producer.workspace = true percent-encoding.workspace = true @@ -56,7 +55,7 @@ reqwest = { workspace = true, features = ["rustls-tls", "stream"] } schemars = { workspace = true, features = [ "chrono", "uuid1" ] } semver.workspace = true serde.workspace = true -serde_json.workspace = true +serde_json = {workspace = true, features = ["raw_value"]} sha3.workspace = true sled-agent-client.workspace = true sled-hardware.workspace = true diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 61d4c84af3..9adfa47d9b 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -7,29 +7,31 @@ use anyhow::{anyhow, Context}; use bootstore::schemes::v0 as bootstore; use ddm_admin_client::{Client as DdmAdminClient, DdmError}; -use dpd_client::types::Ipv6Entry; +use dpd_client::types::{Ipv6Entry, RouteSettingsV6}; use dpd_client::types::{ LinkCreate, LinkId, LinkSettings, PortId, PortSettings, RouteSettingsV4, }; use dpd_client::Client as DpdClient; -use dpd_client::Ipv4Cidr; use futures::future; use gateway_client::Client as MgsClient; use internal_dns::resolver::{ResolveError, Resolver as DnsResolver}; use internal_dns::ServiceName; -use omicron_common::address::{Ipv6Subnet, AZ_PREFIX, MGS_PORT}; +use ipnetwork::{IpNetwork, Ipv6Network}; +use omicron_common::address::{Ipv6Subnet, MGS_PORT}; use omicron_common::address::{DDMD_PORT, DENDRITE_PORT}; use omicron_common::api::internal::shared::{ - PortFec, PortSpeed, RackNetworkConfig, SwitchLocation, UplinkConfig, + PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, RackNetworkConfigV1, + SwitchLocation, UplinkConfig, }; use omicron_common::backoff::{ retry_notify, retry_policy_local, BackoffError, ExponentialBackoff, ExponentialBackoffBuilder, }; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use slog::Logger; use std::collections::{HashMap, HashSet}; -use std::net::{IpAddr, Ipv6Addr, SocketAddrV6}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6}; use std::time::{Duration, Instant}; use thiserror::Error; @@ -107,11 +109,11 @@ impl<'a> EarlyNetworkSetup<'a> { resolver: &DnsResolver, config: &RackNetworkConfig, ) -> HashSet { - // Which switches have uplinks? + // Which switches have configured ports? let uplinked_switches = config - .uplinks + .ports .iter() - .map(|uplink_config| uplink_config.switch) + .map(|port_config| port_config.switch) .collect::>(); // If we have no uplinks, we have nothing to look up. @@ -342,7 +344,7 @@ impl<'a> EarlyNetworkSetup<'a> { &mut self, rack_network_config: &RackNetworkConfig, switch_zone_underlay_ip: Ipv6Addr, - ) -> Result, EarlyNetworkSetupError> { + ) -> Result, EarlyNetworkSetupError> { // First, we have to know which switch we are: ask MGS. info!( self.log, @@ -385,10 +387,10 @@ impl<'a> EarlyNetworkSetup<'a> { }; // We now know which switch we are: filter the uplinks to just ours. - let our_uplinks = rack_network_config - .uplinks + let our_ports = rack_network_config + .ports .iter() - .filter(|uplink| uplink.switch == switch_location) + .filter(|port| port.switch == switch_location) .cloned() .collect::>(); @@ -396,7 +398,7 @@ impl<'a> EarlyNetworkSetup<'a> { self.log, "Initializing {} Uplinks on {switch_location:?} at \ {switch_zone_underlay_ip}", - our_uplinks.len(), + our_ports.len(), ); let dpd = DpdClient::new( &format!("http://[{}]:{}", switch_zone_underlay_ip, DENDRITE_PORT), @@ -408,9 +410,9 @@ impl<'a> EarlyNetworkSetup<'a> { // configure uplink for each requested uplink in configuration that // matches our switch_location - for uplink_config in &our_uplinks { + for port_config in &our_ports { let (ipv6_entry, dpd_port_settings, port_id) = - self.build_uplink_config(uplink_config)?; + self.build_port_config(port_config)?; self.wait_for_dendrite(&dpd).await; @@ -446,14 +448,14 @@ impl<'a> EarlyNetworkSetup<'a> { ddmd_client.advertise_prefix(Ipv6Subnet::new(ipv6_entry.addr)); } - Ok(our_uplinks) + Ok(our_ports) } - fn build_uplink_config( + fn build_port_config( &self, - uplink_config: &UplinkConfig, + port_config: &PortConfigV1, ) -> Result<(Ipv6Entry, PortSettings, PortId), EarlyNetworkSetupError> { - info!(self.log, "Building Uplink Configuration"); + info!(self.log, "Building Port Configuration"); let ipv6_entry = Ipv6Entry { addr: BOUNDARY_SERVICES_ADDR.parse().map_err(|e| { EarlyNetworkSetupError::BadConfig(format!( @@ -469,41 +471,57 @@ impl<'a> EarlyNetworkSetup<'a> { v6_routes: HashMap::new(), }; let link_id = LinkId(0); + + let mut addrs = Vec::new(); + for a in &port_config.addresses { + addrs.push(a.ip()); + } + // TODO We're discarding the `uplink_cidr.prefix()` here and only using // the IP address; at some point we probably need to give the full CIDR // to dendrite? - let addr = IpAddr::V4(uplink_config.uplink_cidr.ip()); let link_settings = LinkSettings { // TODO Allow user to configure link properties // https://github.com/oxidecomputer/omicron/issues/3061 params: LinkCreate { autoneg: false, kr: false, - fec: convert_fec(&uplink_config.uplink_port_fec), - speed: convert_speed(&uplink_config.uplink_port_speed), + fec: convert_fec(&port_config.uplink_port_fec), + speed: convert_speed(&port_config.uplink_port_speed), }, - addrs: vec![addr], + //addrs: vec![addr], + addrs, }; dpd_port_settings.links.insert(link_id.to_string(), link_settings); - let port_id: PortId = - uplink_config.uplink_port.parse().map_err(|e| { - EarlyNetworkSetupError::BadConfig(format!( - concat!( - "could not use value provided to", - "rack_network_config.uplink_port as PortID: {}" - ), - e - )) - })?; - dpd_port_settings.v4_routes.insert( - Ipv4Cidr { prefix: "0.0.0.0".parse().unwrap(), prefix_len: 0 } - .to_string(), - RouteSettingsV4 { - link_id: link_id.0, - vid: uplink_config.uplink_vid, - nexthop: uplink_config.gateway_ip, - }, - ); + let port_id: PortId = port_config.port.parse().map_err(|e| { + EarlyNetworkSetupError::BadConfig(format!( + concat!( + "could not use value provided to", + "rack_network_config.uplink_port as PortID: {}" + ), + e + )) + })?; + + for r in &port_config.routes { + if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) = + (r.destination, r.nexthop) + { + dpd_port_settings.v4_routes.insert( + dst.to_string(), + vec![RouteSettingsV4 { link_id: link_id.0, nexthop }], + ); + } + if let (IpNetwork::V6(dst), IpAddr::V6(nexthop)) = + (r.destination, r.nexthop) + { + dpd_port_settings.v6_routes.insert( + dst.to_string(), + vec![RouteSettingsV6 { link_id: link_id.0, nexthop }], + ); + } + } + Ok((ipv6_entry, dpd_port_settings, port_id)) } @@ -546,33 +564,68 @@ fn retry_policy_switch_mapping() -> ExponentialBackoff { .build() } +// The first production version of the `EarlyNetworkConfig`. +// +// If this version is in the bootstore than we need to convert it to +// `EarlyNetworkConfigV1`. +// +// Once we do this for all customers that have initialized racks with the +// old version we can go ahead and remove this type and its conversion code +// altogether. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +struct EarlyNetworkConfigV0 { + // The current generation number of data as stored in CRDB. + // The initial generation is set during RSS time and then only mutated + // by Nexus. + pub generation: u64, + + pub rack_subnet: Ipv6Addr, + + /// The external NTP server addresses. + pub ntp_servers: Vec, + + // Rack network configuration as delivered from RSS and only existing at + // generation 1 + pub rack_network_config: Option, +} + /// Network configuration required to bring up the control plane /// /// The fields in this structure are those from /// [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This /// is just for the initial rack configuration and cold boot purposes. Updates -/// will come from Nexus in the future. -#[derive(Clone, Debug, Deserialize, Serialize)] +/// come from Nexus. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] pub struct EarlyNetworkConfig { - // The version of data. Always `1` when created from RSS. + // The current generation number of data as stored in CRDB. + // The initial generation is set during RSS time and then only mutated + // by Nexus. pub generation: u64, - pub rack_subnet: Ipv6Addr, + // Which version of the data structure do we have. This is to help with + // deserialization and conversion in future updates. + pub schema_version: u32, + + // The actual configuration details + pub body: EarlyNetworkConfigBody, +} +/// This is the actual configuration of EarlyNetworking. +/// +/// We nest it below the "header" of `generation` and `schema_version` so that +/// we can perform partial deserialization of `EarlyNetworkConfig` to only read +/// the header and defer deserialization of the body once we know the schema +/// version. This is possible via the use of [`serde_json::value::RawValue`] in +/// future (post-v1) deserialization paths. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct EarlyNetworkConfigBody { /// The external NTP server addresses. pub ntp_servers: Vec, - /// A copy of the initial rack network configuration when we are in - /// generation `1`. + // Rack network configuration as delivered from RSS or Nexus pub rack_network_config: Option, } -impl EarlyNetworkConfig { - pub fn az_subnet(&self) -> Ipv6Subnet { - Ipv6Subnet::::new(self.rack_subnet) - } -} - impl From for bootstore::NetworkConfig { fn from(value: EarlyNetworkConfig) -> Self { // Can this ever actually fail? @@ -586,13 +639,77 @@ impl From for bootstore::NetworkConfig { } } +// Note: This currently only converts between v0 and v1 or deserializes v1 of +// `EarlyNetworkConfig`. impl TryFrom for EarlyNetworkConfig { type Error = serde_json::Error; fn try_from( value: bootstore::NetworkConfig, ) -> std::result::Result { - serde_json::from_slice(&value.blob) + // Try to deserialize the latest version of the data structure (v1). If + // that succeeds we are done. + if let Ok(val) = + serde_json::from_slice::(&value.blob) + { + return Ok(val); + } + + // We don't have the latest version. Try to deserialize v0 and then + // convert it to the latest version. + let v0 = serde_json::from_slice::(&value.blob)?; + + Ok(EarlyNetworkConfig { + generation: v0.generation, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: v0.ntp_servers, + rack_network_config: v0.rack_network_config.map(|v0_config| { + RackNetworkConfigV0::to_v1(v0.rack_subnet, v0_config) + }), + }, + }) + } +} + +/// Deprecated, use `RackNetworkConfig` instead. Cannot actually deprecate due to +/// +/// +/// Our first version of `RackNetworkConfig`. If this exists in the bootstore, we +/// upgrade out of it into `RackNetworkConfigV1` or later versions if possible. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct RackNetworkConfigV0 { + // TODO: #3591 Consider making infra-ip ranges implicit for uplinks + /// First ip address to be used for configuring network infrastructure + pub infra_ip_first: Ipv4Addr, + /// Last ip address to be used for configuring network infrastructure + pub infra_ip_last: Ipv4Addr, + /// Uplinks for connecting the rack to external networks + pub uplinks: Vec, +} + +impl RackNetworkConfigV0 { + /// Convert from `RackNetworkConfigV0` to `RackNetworkConfigV1` + /// + /// We cannot use `From for `RackNetworkConfigV1` + /// because the `rack_subnet` field does not exist in `RackNetworkConfigV0` + /// and must be passed in from the `EarlyNetworkConfigV0` struct which + /// contains the `RackNetworkConfivV0` struct. + pub fn to_v1( + rack_subnet: Ipv6Addr, + v0: RackNetworkConfigV0, + ) -> RackNetworkConfigV1 { + RackNetworkConfigV1 { + rack_subnet: Ipv6Network::new(rack_subnet, 56).unwrap(), + infra_ip_first: v0.infra_ip_first, + infra_ip_last: v0.infra_ip_last, + ports: v0 + .uplinks + .into_iter() + .map(|uplink| PortConfigV1::from(uplink)) + .collect(), + bgp: vec![], + } } } @@ -621,3 +738,66 @@ fn convert_fec(fec: &PortFec) -> dpd_client::types::PortFec { PortFec::Rs => dpd_client::types::PortFec::Rs, } } + +#[cfg(test)] +mod tests { + use super::*; + use omicron_common::api::internal::shared::RouteConfig; + + #[test] + fn serialized_early_network_config_v0_to_v1_conversion() { + let v0 = EarlyNetworkConfigV0 { + generation: 1, + rack_subnet: Ipv6Addr::UNSPECIFIED, + ntp_servers: Vec::new(), + rack_network_config: Some(RackNetworkConfigV0 { + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + uplinks: vec![UplinkConfig { + gateway_ip: Ipv4Addr::UNSPECIFIED, + switch: SwitchLocation::Switch0, + uplink_port: "Port0".to_string(), + uplink_port_speed: PortSpeed::Speed100G, + uplink_port_fec: PortFec::None, + uplink_cidr: "192.168.0.1/16".parse().unwrap(), + uplink_vid: None, + }], + }), + }; + + let v0_serialized = serde_json::to_vec(&v0).unwrap(); + let bootstore_conf = + bootstore::NetworkConfig { generation: 1, blob: v0_serialized }; + + let v1 = EarlyNetworkConfig::try_from(bootstore_conf).unwrap(); + let v0_rack_network_config = v0.rack_network_config.unwrap(); + let uplink = v0_rack_network_config.uplinks[0].clone(); + let expected = EarlyNetworkConfig { + generation: 1, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: v0.ntp_servers.clone(), + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: Ipv6Network::new(v0.rack_subnet, 56).unwrap(), + infra_ip_first: v0_rack_network_config.infra_ip_first, + infra_ip_last: v0_rack_network_config.infra_ip_last, + ports: vec![PortConfigV1 { + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: uplink.gateway_ip.into(), + }], + addresses: vec![uplink.uplink_cidr.into()], + switch: uplink.switch, + port: uplink.uplink_port, + uplink_port_speed: uplink.uplink_port_speed, + uplink_port_fec: uplink.uplink_port_fec, + bgp_peers: vec![], + }], + bgp: vec![], + }), + }, + }; + + assert_eq!(expected, v1); + } +} diff --git a/sled-agent/src/bootstrap/maghemite.rs b/sled-agent/src/bootstrap/maghemite.rs index 1adc677b23..2cf0eaf190 100644 --- a/sled-agent/src/bootstrap/maghemite.rs +++ b/sled-agent/src/bootstrap/maghemite.rs @@ -8,7 +8,7 @@ use illumos_utils::addrobj::AddrObject; use slog::Logger; use thiserror::Error; -const SERVICE_FMRI: &str = "svc:/system/illumos/mg-ddm"; +const SERVICE_FMRI: &str = "svc:/oxide/mg-ddm"; const MANIFEST_PATH: &str = "/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"; #[derive(Debug, Error)] diff --git a/sled-agent/src/bootstrap/secret_retriever.rs b/sled-agent/src/bootstrap/secret_retriever.rs index 5cae06310c..1d5ac10ac5 100644 --- a/sled-agent/src/bootstrap/secret_retriever.rs +++ b/sled-agent/src/bootstrap/secret_retriever.rs @@ -14,9 +14,9 @@ use std::sync::OnceLock; static MAYBE_LRTQ_RETRIEVER: OnceLock = OnceLock::new(); -/// A [`key-manager::SecretRetriever`] that either uses a -/// [`LocalSecretRetriever`] or [`LrtqSecretRetriever`] under the hood depending -/// upon how many sleds are in the cluster at rack init time. +/// A [`key_manager::SecretRetriever`] that either uses a +/// [`HardcodedSecretRetriever`] or [`LrtqSecretRetriever`] under the +/// hood depending upon how many sleds are in the cluster at rack init time. pub struct LrtqOrHardcodedSecretRetriever {} impl LrtqOrHardcodedSecretRetriever { diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index 0cbbf0678b..9ed3ad582d 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -528,7 +528,7 @@ fn start_dropshot_server( /// /// TODO-correctness Subsequent steps may assume all M.2s that will ever be /// present are present once we return from this function; see -/// https://github.com/oxidecomputer/omicron/issues/3815. +/// . async fn wait_for_boot_m2(storage_resources: &StorageResources, log: &Logger) { // Wait for at least the M.2 we booted from to show up. loop { diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 2ab8273e39..68330d0c0e 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -5,6 +5,7 @@ //! HTTP entrypoint functions for the sled agent's exposed API use super::sled_agent::SledAgent; +use crate::bootstrap::early_networking::EarlyNetworkConfig; use crate::params::{ CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, @@ -14,6 +15,7 @@ use crate::params::{ }; use crate::sled_agent::Error as SledAgentError; use crate::zone_bundle; +use bootstore::schemes::v0::NetworkConfig; use camino::Utf8PathBuf; use dropshot::{ endpoint, ApiDescription, FreeformBody, HttpError, HttpResponseCreated, @@ -24,9 +26,10 @@ use illumos_utils::opte::params::{ DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost, }; use omicron_common::api::external::Error; -use omicron_common::api::internal::nexus::DiskRuntimeState; -use omicron_common::api::internal::nexus::SledInstanceState; -use omicron_common::api::internal::nexus::UpdateArtifactId; +use omicron_common::api::internal::nexus::{ + DiskRuntimeState, SledInstanceState, UpdateArtifactId, +}; +use omicron_common::api::internal::shared::SwitchPorts; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -62,6 +65,9 @@ pub fn api() -> SledApiDescription { api.register(update_artifact)?; api.register(vpc_firewall_rules_put)?; api.register(zpools_get)?; + api.register(uplink_ensure)?; + api.register(read_network_bootstore_config_cache)?; + api.register(write_network_bootstore_config)?; Ok(()) } @@ -630,3 +636,73 @@ async fn timesync_get( let sa = rqctx.context(); Ok(HttpResponseOk(sa.timesync_get().await.map_err(|e| Error::from(e))?)) } + +#[endpoint { + method = POST, + path = "/switch-ports", +}] +async fn uplink_ensure( + rqctx: RequestContext, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + sa.ensure_scrimlet_host_ports(body.into_inner().uplinks).await?; + Ok(HttpResponseUpdatedNoContent()) +} + +/// This API endpoint is only reading the local sled agent's view of the +/// bootstore. The boostore is a distributed data store that is eventually +/// consistent. Reads from individual nodes may not represent the latest state. +#[endpoint { + method = GET, + path = "/network-bootstore-config", +}] +async fn read_network_bootstore_config_cache( + rqctx: RequestContext, +) -> Result, HttpError> { + let sa = rqctx.context(); + let bs = sa.bootstore(); + + let config = bs.get_network_config().await.map_err(|e| { + HttpError::for_internal_error(format!("failed to get bootstore: {e}")) + })?; + + let config = match config { + Some(config) => EarlyNetworkConfig::try_from(config).map_err(|e| { + HttpError::for_internal_error(format!( + "deserialize early network config: {e}" + )) + })?, + None => { + return Err(HttpError::for_unavail( + None, + "early network config does not exist yet".into(), + )); + } + }; + + Ok(HttpResponseOk(config)) +} + +#[endpoint { + method = PUT, + path = "/network-bootstore-config", +}] +async fn write_network_bootstore_config( + rqctx: RequestContext, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + let bs = sa.bootstore(); + let config = body.into_inner(); + + bs.update_network_config(NetworkConfig::from(config)).await.map_err( + |e| { + HttpError::for_internal_error(format!( + "failed to write updated config to boot store: {e}" + )) + }, + )?; + + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index ce1ef662dc..94614c2363 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -26,7 +26,7 @@ use chrono::Utc; use futures::lock::{Mutex, MutexGuard}; use illumos_utils::dladm::Etherstub; use illumos_utils::link::VnicAllocator; -use illumos_utils::opte::PortManager; +use illumos_utils::opte::{DhcpCfg, PortManager}; use illumos_utils::running_zone::{InstalledZone, RunningZone}; use illumos_utils::svc::wait_for_service; use illumos_utils::zone::Zones; @@ -91,6 +91,10 @@ pub enum Error { #[error(transparent)] Opte(#[from] illumos_utils::opte::Error), + /// Issued by `impl TryFrom<&[u8]> for oxide_vpc::api::DomainName` + #[error("Invalid hostname: {0}")] + InvalidHostname(&'static str), + #[error("Error resolving DNS name: {0}")] ResolveError(#[from] internal_dns::resolver::ResolveError), @@ -207,6 +211,7 @@ struct InstanceInner { source_nat: SourceNatConfig, external_ips: Vec, firewall_rules: Vec, + dhcp_config: DhcpCfg, // Disk related properties // TODO: replace `propolis_client::handmade::*` with properly-modeled local types @@ -610,6 +615,37 @@ impl Instance { zone_bundler, } = services; + let mut dhcp_config = DhcpCfg { + hostname: Some( + hardware + .properties + .hostname + .parse() + .map_err(Error::InvalidHostname)?, + ), + host_domain: hardware + .dhcp_config + .host_domain + .map(|domain| domain.parse()) + .transpose() + .map_err(Error::InvalidHostname)?, + domain_search_list: hardware + .dhcp_config + .search_domains + .into_iter() + .map(|domain| domain.parse()) + .collect::>() + .map_err(Error::InvalidHostname)?, + dns4_servers: Vec::new(), + dns6_servers: Vec::new(), + }; + for ip in hardware.dhcp_config.dns_servers { + match ip { + IpAddr::V4(ip) => dhcp_config.dns4_servers.push(ip.into()), + IpAddr::V6(ip) => dhcp_config.dns6_servers.push(ip.into()), + } + } + let instance = InstanceInner { log: log.new(o!("instance_id" => id.to_string())), // NOTE: Mostly lies. @@ -633,6 +669,7 @@ impl Instance { source_nat: hardware.source_nat, external_ips: hardware.external_ips, firewall_rules: hardware.firewall_rules, + dhcp_config, requested_disks: hardware.disks, cloud_init_bytes: hardware.cloud_init_bytes, state: InstanceStates::new( @@ -852,6 +889,7 @@ impl Instance { snat, external_ips, &inner.firewall_rules, + inner.dhcp_config.clone(), )?; opte_ports.push(port); } diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 84ec1ef0dc..5fda3c1ae6 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -6,6 +6,7 @@ use crate::zone_bundle::PriorityOrder; pub use crate::zone_bundle::ZoneBundleCause; pub use crate::zone_bundle::ZoneBundleId; pub use crate::zone_bundle::ZoneBundleMetadata; +pub use illumos_utils::opte::params::DhcpConfig; pub use illumos_utils::opte::params::VpcFirewallRule; pub use illumos_utils::opte::params::VpcFirewallRulesEnsureBody; use omicron_common::api::internal::nexus::{ @@ -68,6 +69,7 @@ pub struct InstanceHardware { /// provided to an instance to allow inbound connectivity. pub external_ips: Vec, pub firewall_rules: Vec, + pub dhcp_config: DhcpConfig, // TODO: replace `propolis_client::handmade::*` with locally-modeled request type pub disks: Vec, pub cloud_init_bytes: Option, @@ -349,10 +351,12 @@ pub enum ServiceType { #[serde(skip)] Uplink, #[serde(skip)] - Maghemite { + MgDdm { mode: String, }, #[serde(skip)] + Mgd, + #[serde(skip)] SpSim, CruciblePantry { address: SocketAddrV6, @@ -402,7 +406,8 @@ impl std::fmt::Display for ServiceType { ServiceType::CruciblePantry { .. } => write!(f, "crucible/pantry"), ServiceType::BoundaryNtp { .. } | ServiceType::InternalNtp { .. } => write!(f, "ntp"), - ServiceType::Maghemite { .. } => write!(f, "mg-ddm"), + ServiceType::MgDdm { .. } => write!(f, "mg-ddm"), + ServiceType::Mgd => write!(f, "mgd"), ServiceType::SpSim => write!(f, "sp-sim"), ServiceType::Clickhouse { .. } => write!(f, "clickhouse"), ServiceType::ClickhouseKeeper { .. } => { @@ -419,13 +424,7 @@ impl crate::smf_helper::Service for ServiceType { self.to_string() } fn smf_name(&self) -> String { - match self { - // NOTE: This style of service-naming is deprecated - ServiceType::Maghemite { .. } => { - format!("svc:/system/illumos/{}", self.service_name()) - } - _ => format!("svc:/oxide/{}", self.service_name()), - } + format!("svc:/oxide/{}", self.service_name()) } fn should_import(&self) -> bool { true @@ -525,7 +524,8 @@ impl TryFrom for sled_agent_client::types::ServiceType { | St::Dendrite { .. } | St::Tfport { .. } | St::Uplink - | St::Maghemite { .. } => Err(AutonomousServiceOnlyError), + | St::Mgd + | St::MgDdm { .. } => Err(AutonomousServiceOnlyError), } } } @@ -824,7 +824,8 @@ impl ServiceZoneRequest { | ServiceType::SpSim | ServiceType::Wicketd { .. } | ServiceType::Dendrite { .. } - | ServiceType::Maghemite { .. } + | ServiceType::MgDdm { .. } + | ServiceType::Mgd | ServiceType::Tfport { .. } | ServiceType::Uplink => { return Err(AutonomousServiceOnlyError); diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 2183aa7b63..3dac5d7d1e 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -19,10 +19,11 @@ use internal_dns::{ServiceName, DNS_ZONE}; use omicron_common::address::{ get_sled_address, get_switch_zone_address, Ipv6Subnet, ReservedRackSubnet, DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, DNS_REDUNDANCY, MAX_DNS_REDUNDANCY, - MGS_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, RSS_RESERVED_ADDRESSES, + MGD_PORT, MGS_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, RSS_RESERVED_ADDRESSES, SLED_PREFIX, }; use omicron_common::api::external::{MacAddr, Vni}; +use omicron_common::api::internal::shared::SwitchLocation; use omicron_common::api::internal::shared::{ NetworkInterface, NetworkInterfaceKind, SourceNatConfig, }; @@ -276,7 +277,7 @@ impl Plan { "No scrimlets observed".to_string(), )); } - for sled in scrimlets { + for (i, sled) in scrimlets.iter().enumerate() { let address = get_switch_zone_address(sled.subnet); let zone = dns_builder.host_dendrite(sled.sled_id, address).unwrap(); @@ -294,6 +295,18 @@ impl Plan { MGS_PORT, ) .unwrap(); + dns_builder + .service_backend_zone(ServiceName::Mgd, &zone, MGD_PORT) + .unwrap(); + + // TODO only works for single rack + let sled_address = get_sled_address(sled.subnet); + let switch_location = if i == 0 { + SwitchLocation::Switch0 + } else { + SwitchLocation::Switch1 + }; + dns_builder.host_scrimlet(switch_location, sled_address).unwrap(); } // We'll stripe most services across all available Sleds, round-robin diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 805c889295..7f6469d2c0 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -57,7 +57,8 @@ use super::config::SetupServiceConfig as Config; use crate::bootstrap::config::BOOTSTRAP_AGENT_HTTP_PORT; use crate::bootstrap::early_networking::{ - EarlyNetworkConfig, EarlyNetworkSetup, EarlyNetworkSetupError, + EarlyNetworkConfig, EarlyNetworkConfigBody, EarlyNetworkSetup, + EarlyNetworkSetupError, }; use crate::bootstrap::params::BootstrapAddressDiscovery; use crate::bootstrap::params::StartSledAgentRequest; @@ -575,17 +576,25 @@ impl ServiceInner { let rack_network_config = match &config.rack_network_config { Some(config) => { - let value = NexusTypes::RackNetworkConfig { + let value = NexusTypes::RackNetworkConfigV1 { + rack_subnet: config.rack_subnet, infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, - uplinks: config - .uplinks + ports: config + .ports .iter() - .map(|config| NexusTypes::UplinkConfig { - gateway_ip: config.gateway_ip, + .map(|config| NexusTypes::PortConfigV1 { + port: config.port.clone(), + routes: config + .routes + .iter() + .map(|r| NexusTypes::RouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), switch: config.switch.into(), - uplink_cidr: config.uplink_cidr, - uplink_port: config.uplink_port.clone(), uplink_port_speed: config .uplink_port_speed .clone() @@ -594,7 +603,23 @@ impl ServiceInner { .uplink_port_fec .clone() .into(), - uplink_vid: config.uplink_vid, + bgp_peers: config + .bgp_peers + .iter() + .map(|b| NexusTypes::BgpPeerConfig { + addr: b.addr, + asn: b.asn, + port: b.port.clone(), + }) + .collect(), + }) + .collect(), + bgp: config + .bgp + .iter() + .map(|config| NexusTypes::BgpConfig { + asn: config.asn, + originate: config.originate.clone(), }) .collect(), }; @@ -872,9 +897,11 @@ impl ServiceInner { // from the bootstore". let early_network_config = EarlyNetworkConfig { generation: 1, - rack_subnet: config.rack_subnet, - ntp_servers: config.ntp_servers.clone(), - rack_network_config: config.rack_network_config.clone(), + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: config.ntp_servers.clone(), + rack_network_config: config.rack_network_config.clone(), + }, }; info!(self.log, "Writing Rack Network Configuration to bootstore"); bootstore.update_network_config(early_network_config.into()).await?; diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 60f0965612..a9be0e7c4a 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -52,7 +52,7 @@ use illumos_utils::dladm::{ Dladm, Etherstub, EtherstubVnic, GetSimnetError, PhysicalLink, }; use illumos_utils::link::{Link, VnicAllocator}; -use illumos_utils::opte::{Port, PortManager, PortTicket}; +use illumos_utils::opte::{DhcpCfg, Port, PortManager, PortTicket}; use illumos_utils::running_zone::{ InstalledZone, RunCommandError, RunningZone, }; @@ -62,7 +62,6 @@ use illumos_utils::zone::Zones; use illumos_utils::{execute, PFEXEC}; use internal_dns::resolver::Resolver; use itertools::Itertools; -use omicron_common::address::Ipv6Subnet; use omicron_common::address::AZ_PREFIX; use omicron_common::address::BOOTSTRAP_ARTIFACT_PORT; use omicron_common::address::CLICKHOUSE_KEEPER_PORT; @@ -74,9 +73,13 @@ use omicron_common::address::DENDRITE_PORT; use omicron_common::address::MGS_PORT; use omicron_common::address::RACK_PREFIX; use omicron_common::address::SLED_PREFIX; +use omicron_common::address::WICKETD_NEXUS_PROXY_PORT; use omicron_common::address::WICKETD_PORT; +use omicron_common::address::{Ipv6Subnet, NEXUS_TECHPORT_EXTERNAL_PORT}; use omicron_common::api::external::Generation; -use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::{ + HostPortConfig, RackNetworkConfig, +}; use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, retry_policy_local, BackoffError, @@ -95,8 +98,8 @@ use sled_hardware::underlay::BOOTSTRAP_PREFIX; use sled_hardware::Baseboard; use sled_hardware::SledMode; use slog::Logger; +use std::collections::BTreeMap; use std::collections::HashSet; -use std::collections::{BTreeMap, HashMap}; use std::iter; use std::iter::FromIterator; use std::net::{IpAddr, Ipv6Addr, SocketAddr}; @@ -757,7 +760,7 @@ impl ServiceManager { } } } - ServiceType::Maghemite { .. } => { + ServiceType::MgDdm { .. } => { // If on a non-gimlet, sled-agent can be configured to map // links into the switch zone. Validate those links here. for link in &self.inner.switch_zone_maghemite_links { @@ -862,11 +865,11 @@ impl ServiceManager { // config allows outbound access which is enough for // Boundary NTP which needs to come up before Nexus. let port = port_manager - .create_port(nic, snat, external_ips, &[]) + .create_port(nic, snat, external_ips, &[], DhcpCfg::default()) .map_err(|err| Error::ServicePortCreation { - service: svc.details.to_string(), - err: Box::new(err), - })?; + service: svc.details.to_string(), + err: Box::new(err), + })?; // We also need to update the switch with the NAT mappings let (target_ip, first_port, last_port) = match snat { @@ -1448,6 +1451,8 @@ impl ServiceManager { let deployment_config = NexusDeploymentConfig { id: request.zone.id, rack_id: sled_info.rack_id, + techport_external_server_port: + NEXUS_TECHPORT_EXTERNAL_PORT, dropshot_external: ConfigDropshotWithTls { tls: *external_tls, @@ -1696,6 +1701,28 @@ impl ServiceManager { &format!("[::1]:{MGS_PORT}"), )?; + // We intentionally bind `nexus-proxy-address` to `::` so + // wicketd will serve this on all interfaces, particularly + // the tech port interfaces, allowing external clients to + // connect to this Nexus proxy. + smfh.setprop( + "config/nexus-proxy-address", + &format!("[::]:{WICKETD_NEXUS_PROXY_PORT}"), + )?; + if let Some(underlay_address) = self + .inner + .sled_info + .get() + .map(|info| info.underlay_address) + { + let rack_subnet = + Ipv6Subnet::::new(underlay_address); + smfh.setprop( + "config/rack-subnet", + &rack_subnet.net().ip().to_string(), + )?; + } + let serialized_baseboard = serde_json::to_string_pretty(&baseboard)?; let serialized_baseboard_path = Utf8PathBuf::from(format!( @@ -1928,8 +1955,13 @@ impl ServiceManager { // Nothing to do here - this service is special and // configured in `ensure_switch_zone_uplinks_configured` } - ServiceType::Maghemite { mode } => { - info!(self.inner.log, "Setting up Maghemite service"); + ServiceType::Mgd => { + info!(self.inner.log, "Setting up mgd service"); + smfh.setprop("config/admin_host", "::")?; + smfh.refresh()?; + } + ServiceType::MgDdm { mode } => { + info!(self.inner.log, "Setting up mg-ddm service"); smfh.setprop("config/mode", &mode)?; smfh.setprop("config/admin_host", "::")?; @@ -1990,8 +2022,8 @@ impl ServiceManager { )?; if is_gimlet { - // Maghemite for a scrimlet needs to be configured to - // talk to dendrite + // Ddm for a scrimlet needs to be configured to talk to + // dendrite smfh.setprop("config/dpd_host", "[::1]")?; smfh.setprop("config/dpd_port", DENDRITE_PORT)?; } @@ -2480,7 +2512,8 @@ impl ServiceManager { ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, ServiceType::Uplink, ServiceType::Wicketd { baseboard }, - ServiceType::Maghemite { mode: "transit".to_string() }, + ServiceType::Mgd, + ServiceType::MgDdm { mode: "transit".to_string() }, ] } @@ -2503,7 +2536,8 @@ impl ServiceManager { ServiceType::ManagementGatewayService, ServiceType::Uplink, ServiceType::Wicketd { baseboard }, - ServiceType::Maghemite { mode: "transit".to_string() }, + ServiceType::Mgd, + ServiceType::MgDdm { mode: "transit".to_string() }, ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, ServiceType::SpSim, ] @@ -2558,10 +2592,20 @@ impl ServiceManager { let log = &self.inner.log; // Configure uplinks via DPD in our switch zone. - let our_uplinks = EarlyNetworkSetup::new(log) + let our_ports = EarlyNetworkSetup::new(log) .init_switch_config(rack_network_config, switch_zone_ip) - .await?; + .await? + .into_iter() + .map(From::from) + .collect(); + + self.ensure_scrimlet_host_ports(our_ports).await + } + pub async fn ensure_scrimlet_host_ports( + &self, + our_ports: Vec, + ) -> Result<(), Error> { // We expect the switch zone to be running, as we're called immediately // after `ensure_zone()` above and we just successfully configured // uplinks via DPD running in our switch zone. If somehow we're in any @@ -2592,22 +2636,14 @@ impl ServiceManager { smfh.delpropgroup("uplinks")?; smfh.addpropgroup("uplinks", "application")?; - // When naming the uplink ports, we need to append `_0`, `_1`, etc., for - // each use of any given port. We use a hashmap of counters of port name - // -> number of uplinks to correctly supply that suffix. - let mut port_count = HashMap::new(); - for uplink_config in &our_uplinks { - let this_port_count: &mut usize = - port_count.entry(&uplink_config.uplink_port).or_insert(0); - smfh.addpropvalue_type( - &format!( - "uplinks/{}_{}", - uplink_config.uplink_port, *this_port_count - ), - &uplink_config.uplink_cidr.to_string(), - "astring", - )?; - *this_port_count += 1; + for port_config in &our_ports { + for addr in &port_config.addrs { + smfh.addpropvalue_type( + &format!("uplinks/{}_0", port_config.port,), + &addr.to_string(), + "astring", + )?; + } } smfh.refresh()?; @@ -2705,9 +2741,8 @@ impl ServiceManager { ); *request = new_request; - let address = request - .addresses - .get(0) + let first_address = request.addresses.get(0); + let address = first_address .map(|addr| addr.to_string()) .unwrap_or_else(|| "".to_string()); @@ -2813,6 +2848,29 @@ impl ServiceManager { } smfh.refresh()?; } + ServiceType::Wicketd { .. } => { + if let Some(&address) = first_address { + let rack_subnet = + Ipv6Subnet::::new(address); + + info!( + self.inner.log, "configuring wicketd"; + "rack_subnet" => %rack_subnet.net().ip(), + ); + + smfh.setprop( + "config/rack-subnet", + &rack_subnet.net().ip().to_string(), + )?; + + smfh.refresh()?; + } else { + error!( + self.inner.log, + "underlay address unexpectedly missing", + ); + } + } ServiceType::Tfport { .. } => { // Since tfport and dpd communicate using localhost, // the tfport service shouldn't need to be restarted. @@ -2821,7 +2879,7 @@ impl ServiceManager { // Only configured in // `ensure_switch_zone_uplinks_configured` } - ServiceType::Maghemite { mode } => { + ServiceType::MgDdm { mode } => { smfh.delpropvalue("config/mode", "*")?; smfh.addpropvalue("config/mode", &mode)?; smfh.refresh()?; diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 08f6c7d10b..f77da11b0e 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -4,6 +4,9 @@ //! HTTP entrypoint functions for the sled agent's exposed API +use crate::bootstrap::early_networking::{ + EarlyNetworkConfig, EarlyNetworkConfigBody, +}; use crate::params::{ DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, InstancePutStateResponse, InstanceUnregisterResponse, @@ -19,11 +22,15 @@ use dropshot::RequestContext; use dropshot::TypedBody; use illumos_utils::opte::params::DeleteVirtualNetworkInterfaceHost; use illumos_utils::opte::params::SetVirtualNetworkInterfaceHost; +use ipnetwork::Ipv6Network; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::UpdateArtifactId; +use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::SwitchPorts; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::net::{Ipv4Addr, Ipv6Addr}; use std::sync::Arc; use uuid::Uuid; @@ -46,6 +53,9 @@ pub fn api() -> SledApiDescription { api.register(vpc_firewall_rules_put)?; api.register(set_v2p)?; api.register(del_v2p)?; + api.register(uplink_ensure)?; + api.register(read_network_bootstore_config)?; + api.register(write_network_bootstore_config)?; Ok(()) } @@ -327,3 +337,50 @@ async fn del_v2p( Ok(HttpResponseUpdatedNoContent()) } + +#[endpoint { + method = POST, + path = "/switch-ports", +}] +async fn uplink_ensure( + _rqctx: RequestContext>, + _body: TypedBody, +) -> Result { + Ok(HttpResponseUpdatedNoContent()) +} + +#[endpoint { + method = GET, + path = "/network-bootstore-config", +}] +async fn read_network_bootstore_config( + _rqctx: RequestContext>, +) -> Result, HttpError> { + let config = EarlyNetworkConfig { + generation: 0, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: Vec::new(), + rack_network_config: Some(RackNetworkConfig { + rack_subnet: Ipv6Network::new(Ipv6Addr::UNSPECIFIED, 56) + .unwrap(), + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports: Vec::new(), + bgp: Vec::new(), + }), + }, + }; + Ok(HttpResponseOk(config)) +} + +#[endpoint { + method = PUT, + path = "/network-bootstore-config", +}] +async fn write_network_bootstore_config( + _rqctx: RequestContext>, + _body: TypedBody, +) -> Result { + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 595d83a7ee..1f2fe8e1d8 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -94,7 +94,7 @@ impl Server { &config.id, &NexusTypes::SledAgentStartupInfo { sa_address: sa_address.to_string(), - role: NexusTypes::SledRole::Gimlet, + role: NexusTypes::SledRole::Scrimlet, baseboard: NexusTypes::Baseboard { serial_number: format!( "sim-{}", diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index b6f910220e..52513f081d 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -38,7 +38,9 @@ use omicron_common::api::external::Vni; use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; -use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::{ + HostPortConfig, RackNetworkConfig, +}; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, internal::nexus::UpdateArtifactId, @@ -237,6 +239,9 @@ struct SledAgentInner { // Object managing zone bundles. zone_bundler: zone_bundle::ZoneBundler, + + // A handle to the bootstore. + bootstore: bootstore::NodeHandle, } impl SledAgentInner { @@ -407,7 +412,7 @@ impl SledAgent { EarlyNetworkConfig::try_from(serialized_config) .map_err(|err| BackoffError::transient(err.to_string()))?; - Ok(early_network_config.rack_network_config) + Ok(early_network_config.body.rack_network_config) }; let rack_network_config: Option = retry_notify::<_, String, _, _, _, _>( @@ -458,6 +463,7 @@ impl SledAgent { nexus_request_queue: NexusRequestQueue::new(), rack_network_config, zone_bundler, + bootstore: bootstore.clone(), }), log: log.clone(), }; @@ -769,7 +775,7 @@ impl SledAgent { /// Idempotently ensures that a given instance is registered with this sled, /// i.e., that it can be addressed by future calls to - /// [`instance_ensure_state`]. + /// [`Self::instance_ensure_state`]. pub async fn instance_ensure_registered( &self, instance_id: Uuid, @@ -918,4 +924,19 @@ impl SledAgent { pub async fn timesync_get(&self) -> Result { self.inner.services.timesync_get().await.map_err(Error::from) } + + pub async fn ensure_scrimlet_host_ports( + &self, + uplinks: Vec, + ) -> Result<(), Error> { + self.inner + .services + .ensure_scrimlet_host_ports(uplinks) + .await + .map_err(Error::from) + } + + pub fn bootstore(&self) -> bootstore::NodeHandle { + self.inner.bootstore.clone() + } } diff --git a/smf/sled-agent/gimlet-standalone/config-rss.toml b/smf/sled-agent/gimlet-standalone/config-rss.toml index c6fbab49de..29a7a79eba 100644 --- a/smf/sled-agent/gimlet-standalone/config-rss.toml +++ b/smf/sled-agent/gimlet-standalone/config-rss.toml @@ -88,27 +88,34 @@ last = "192.168.1.29" # Configuration to bring up Boundary Services and make Nexus reachable from the # outside. See docs/how-to-run.adoc for more on what to put here. [rack_network_config] +rack_subnet = "fd00:1122:3344:01::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. infra_ip_first = "192.168.1.30" infra_ip_last = "192.168.1.30" +# Configurations for BGP routers to run on the scrimlets. +bgp = [] + # You can configure multiple uplinks by repeating the following stanza -[[rack_network_config.uplinks]] -# The gateway IP for the rack's external network -gateway_ip = "192.168.1.199" +[[rack_network_config.ports]] +# Routes associated with this port. +routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] +# Addresses associated with this port. +addresses = ["192.168.1.30/32"] # Name of the uplink port. This should always be "qsfp0" when using softnpu. -uplink_port = "qsfp0" +port = "qsfp0" +# The speed of this port. uplink_port_speed = "40G" +# The forward error correction mode for this port. uplink_port_fec="none" -# For softnpu, an address within the "infra" block above that will be used for -# the softnpu uplink port. You can just pick the first address in that pool. -uplink_cidr = "192.168.1.30/32" # Switch to use for the uplink. For single-rack deployments this can be # "switch0" (upper slot) or "switch1" (lower slot). For single-node softnpu # and dendrite stub environments, use "switch0" switch = "switch0" +# Neighbors we expect to peer with over BGP on this port. +bgp_peers = [] # Configuration for the initial Silo, user, and password. # diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index 8a009dd687..fea3cfa5d8 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -88,27 +88,34 @@ last = "192.168.1.29" # Configuration to bring up Boundary Services and make Nexus reachable from the # outside. See docs/how-to-run.adoc for more on what to put here. [rack_network_config] +rack_subnet = "fd00:1122:3344:01::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. infra_ip_first = "192.168.1.30" infra_ip_last = "192.168.1.30" +# Configurations for BGP routers to run on the scrimlets. +bgp = [] + # You can configure multiple uplinks by repeating the following stanza -[[rack_network_config.uplinks]] -# The gateway IP for the rack's external network -gateway_ip = "192.168.1.199" +[[rack_network_config.ports]] +# Routes associated with this port. +routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] +# Addresses associated with this port. +addresses = ["192.168.1.30/32"] # Name of the uplink port. This should always be "qsfp0" when using softnpu. -uplink_port = "qsfp0" +port = "qsfp0" +# The speed of this port. uplink_port_speed = "40G" +# The forward error correction mode for this port. uplink_port_fec="none" -# For softnpu, an address within the "infra" block above that will be used for -# the softnpu uplink port. You can just pick the first address in that pool. -uplink_cidr = "192.168.1.30/32" # Switch to use for the uplink. For single-rack deployments this can be # "switch0" (upper slot) or "switch1" (lower slot). For single-node softnpu # and dendrite stub environments, use "switch0" switch = "switch0" +# Neighbors we expect to peer with over BGP on this port. +bgp_peers = [] # Configuration for the initial Silo, user, and password. # diff --git a/smf/wicketd/manifest.xml b/smf/wicketd/manifest.xml index cb7ed657e0..778a7abf2d 100644 --- a/smf/wicketd/manifest.xml +++ b/smf/wicketd/manifest.xml @@ -12,10 +12,29 @@ + + + @@ -24,7 +43,16 @@ + + + + diff --git a/sp-sim/src/gimlet.rs b/sp-sim/src/gimlet.rs index dad53f3848..d131696559 100644 --- a/sp-sim/src/gimlet.rs +++ b/sp-sim/src/gimlet.rs @@ -1282,6 +1282,25 @@ impl SpHandler for Handler { buf[..val.len()].copy_from_slice(val); Ok(val.len()) } + + fn read_sensor( + &mut self, + _request: gateway_messages::SensorRequest, + ) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } + + fn current_time(&mut self) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } + + fn read_rot( + &mut self, + _request: gateway_messages::RotRequest, + _buf: &mut [u8], + ) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } } enum UpdateState { diff --git a/sp-sim/src/sidecar.rs b/sp-sim/src/sidecar.rs index def2a79c0c..e56c610c9c 100644 --- a/sp-sim/src/sidecar.rs +++ b/sp-sim/src/sidecar.rs @@ -1001,6 +1001,25 @@ impl SpHandler for Handler { buf[..val.len()].copy_from_slice(val); Ok(val.len()) } + + fn read_sensor( + &mut self, + _request: gateway_messages::SensorRequest, + ) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } + + fn current_time(&mut self) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } + + fn read_rot( + &mut self, + _request: gateway_messages::RotRequest, + _buf: &mut [u8], + ) -> std::result::Result { + Err(SpError::RequestUnsupportedForSp) + } } struct FakeIgnition { diff --git a/test-utils/src/dev/clickhouse.rs b/test-utils/src/dev/clickhouse.rs index 8e6920f0be..e96f969bbc 100644 --- a/test-utils/src/dev/clickhouse.rs +++ b/test-utils/src/dev/clickhouse.rs @@ -8,7 +8,8 @@ use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Duration; -use anyhow::Context; +use anyhow::{anyhow, Context}; +use std::net::SocketAddr; use tempfile::{Builder, TempDir}; use thiserror::Error; use tokio::{ @@ -20,8 +21,7 @@ use tokio::{ use crate::dev::poll; // Timeout used when starting up ClickHouse subprocess. -// build-and-test (ubuntu-20.04) needs a little longer to get going -const CLICKHOUSE_TIMEOUT: Duration = Duration::from_secs(90); +const CLICKHOUSE_TIMEOUT: Duration = Duration::from_secs(30); /// A `ClickHouseInstance` is used to start and manage a ClickHouse single node server process. #[derive(Debug)] @@ -31,12 +31,24 @@ pub struct ClickHouseInstance { data_path: PathBuf, // The HTTP port the server is listening on port: u16, + // The address the server is listening on + pub address: Option, // Full list of command-line arguments args: Vec, // Subprocess handle child: Option, } +/// A `ClickHouseCluster` is used to start and manage a 2 replica 3 keeper ClickHouse cluster. +#[derive(Debug)] +pub struct ClickHouseCluster { + pub replica_1: ClickHouseInstance, + pub replica_2: ClickHouseInstance, + pub keeper_1: ClickHouseInstance, + pub keeper_2: ClickHouseInstance, + pub keeper_3: ClickHouseInstance, +} + #[derive(Debug, Error)] pub enum ClickHouseError { #[error("Failed to open ClickHouse log file")] @@ -101,6 +113,7 @@ impl ClickHouseInstance { data_dir: Some(data_dir), data_path, port, + address: None, args, child: Some(child), }) @@ -162,6 +175,7 @@ impl ClickHouseInstance { })?; let data_path = data_dir.path().to_path_buf(); + let address = SocketAddr::new("127.0.0.1".parse().unwrap(), port); let result = wait_for_ready(log_path).await; match result { @@ -169,6 +183,7 @@ impl ClickHouseInstance { data_dir: Some(data_dir), data_path, port, + address: Some(address), args, child: Some(child), }), @@ -244,6 +259,7 @@ impl ClickHouseInstance { data_dir: Some(data_dir), data_path, port, + address: None, args, child: Some(child), }), @@ -314,6 +330,97 @@ impl Drop for ClickHouseInstance { } } +impl ClickHouseCluster { + pub async fn new() -> Result { + // Start all Keeper coordinator nodes + let cur_dir = std::env::current_dir().unwrap(); + let keeper_config = + cur_dir.as_path().join("src/configs/keeper_config.xml"); + + let keeper_amount = 3; + let mut keepers = + Self::new_keeper_set(keeper_amount, keeper_config).await?; + + // Start all replica nodes + let cur_dir = std::env::current_dir().unwrap(); + let replica_config = + cur_dir.as_path().join("src/configs/replica_config.xml"); + + let replica_amount = 2; + let mut replicas = + Self::new_replica_set(replica_amount, replica_config).await?; + + let r1 = replicas.swap_remove(0); + let r2 = replicas.swap_remove(0); + let k1 = keepers.swap_remove(0); + let k2 = keepers.swap_remove(0); + let k3 = keepers.swap_remove(0); + + Ok(Self { + replica_1: r1, + replica_2: r2, + keeper_1: k1, + keeper_2: k2, + keeper_3: k3, + }) + } + + pub async fn new_keeper_set( + keeper_amount: u16, + config_path: PathBuf, + ) -> Result, anyhow::Error> { + let mut keepers = vec![]; + + for i in 1..=keeper_amount { + let k_port = 9180 + i; + let k_id = i; + + let k = ClickHouseInstance::new_keeper( + k_port, + k_id, + config_path.clone(), + ) + .await + .map_err(|e| { + anyhow!("Failed to start ClickHouse keeper {}: {}", i, e) + })?; + keepers.push(k) + } + + Ok(keepers) + } + + pub async fn new_replica_set( + replica_amount: u16, + config_path: PathBuf, + ) -> Result, anyhow::Error> { + let mut replicas = vec![]; + + for i in 1..=replica_amount { + let r_port = 8122 + i; + let r_tcp_port = 9000 + i; + let r_interserver_port = 9008 + i; + let r_name = format!("oximeter_cluster node {}", i); + let r_number = format!("0{}", i); + let r = ClickHouseInstance::new_replicated( + r_port, + r_tcp_port, + r_interserver_port, + r_name, + r_number, + config_path.clone(), + ) + .await + .map_err(|e| { + anyhow!("Failed to start ClickHouse node {}: {}", i, e) + })?; + replicas.push(r) + } + + Ok(replicas) + } +} + // Wait for the ClickHouse log file to become available, including the // port number. // diff --git a/test-utils/src/dev/dendrite.rs b/test-utils/src/dev/dendrite.rs index 520bf12401..8938595aa2 100644 --- a/test-utils/src/dev/dendrite.rs +++ b/test-utils/src/dev/dendrite.rs @@ -19,7 +19,7 @@ use tokio::{ /// Specifies the amount of time we will wait for `dpd` to launch, /// which is currently confirmed by watching `dpd`'s log output /// for a message specifying the address and port `dpd` is listening on. -pub const DENDRITE_TIMEOUT: Duration = Duration::new(5, 0); +pub const DENDRITE_TIMEOUT: Duration = Duration::new(30, 0); /// Represents a running instance of the Dendrite dataplane daemon (dpd). pub struct DendriteInstance { diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs new file mode 100644 index 0000000000..fa1f353896 --- /dev/null +++ b/test-utils/src/dev/maghemite.rs @@ -0,0 +1,155 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tools for managing Maghemite during development + +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::Context; +use tempfile::TempDir; +use tokio::{ + fs::File, + io::{AsyncBufReadExt, BufReader}, + time::{sleep, Instant}, +}; + +/// Specifies the amount of time we will wait for `mgd` to launch, +/// which is currently confirmed by watching `mgd`'s log output +/// for a message specifying the address and port `mgd` is listening on. +pub const MGD_TIMEOUT: Duration = Duration::new(5, 0); + +pub struct MgdInstance { + /// Port number the mgd instance is listening on. This can be provided + /// manually, or dynamically determined if a value of 0 is provided. + pub port: u16, + /// Arguments provided to the `mgd` cli command. + pub args: Vec, + /// Child process spawned by running `mgd` + pub child: Option, + /// Temporary directory where logging output and other files generated by + /// `mgd` are stored. + pub data_dir: Option, +} + +impl MgdInstance { + pub async fn start(mut port: u16) -> Result { + let temp_dir = TempDir::new()?; + + let args = vec![ + "run".to_string(), + "--admin-addr".into(), + "::1".into(), + "--admin-port".into(), + port.to_string(), + "--no-bgp-dispatcher".into(), + "--data-dir".into(), + temp_dir.path().display().to_string(), + ]; + + let child = tokio::process::Command::new("mgd") + .args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::from(redirect_file(temp_dir.path(), "mgd_stdout")?)) + .stderr(Stdio::from(redirect_file(temp_dir.path(), "mgd_stderr")?)) + .spawn() + .with_context(|| { + format!("failed to spawn `mgd` (with args: {:?})", &args) + })?; + + let child = Some(child); + + let temp_dir = temp_dir.into_path(); + if port == 0 { + port = discover_port( + temp_dir.join("mgd_stdout").display().to_string(), + ) + .await + .with_context(|| { + format!( + "failed to discover mgd port from files in {}", + temp_dir.display() + ) + })?; + } + + Ok(Self { port, args, child, data_dir: Some(temp_dir) }) + } + + pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { + if let Some(mut child) = self.child.take() { + child.start_kill().context("Sending SIGKILL to child")?; + child.wait().await.context("waiting for child")?; + } + if let Some(dir) = self.data_dir.take() { + std::fs::remove_dir_all(&dir).with_context(|| { + format!("cleaning up temporary directory {}", dir.display()) + })?; + } + Ok(()) + } +} + +impl Drop for MgdInstance { + fn drop(&mut self) { + if self.child.is_some() || self.data_dir.is_some() { + eprintln!( + "WARN: dropped MgdInstance without cleaning it up first \ + (there may still be a child process running and a \ + temporary directory leaked)" + ); + if let Some(child) = self.child.as_mut() { + let _ = child.start_kill(); + } + if let Some(path) = self.data_dir.take() { + eprintln!( + "WARN: mgd temporary directory leaked: {}", + path.display() + ); + } + } + } +} + +fn redirect_file( + temp_dir_path: &Path, + label: &str, +) -> Result { + let out_path = temp_dir_path.join(label); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&out_path) + .with_context(|| format!("open \"{}\"", out_path.display())) +} + +async fn discover_port(logfile: String) -> Result { + let timeout = Instant::now() + MGD_TIMEOUT; + tokio::time::timeout_at(timeout, find_mgd_port_in_log(logfile)) + .await + .context("time out while discovering mgd port number")? +} + +async fn find_mgd_port_in_log(logfile: String) -> Result { + let re = regex::Regex::new(r#""local_addr":"\[::1\]:?([0-9]+)""#).unwrap(); + let reader = BufReader::new(File::open(logfile).await?); + let mut lines = reader.lines(); + loop { + match lines.next_line().await? { + Some(line) => { + if let Some(cap) = re.captures(&line) { + // unwrap on get(1) should be ok, since captures() returns + // `None` if there are no matches found + let port = cap.get(1).unwrap(); + let result = port.as_str().parse::()?; + return Ok(result); + } + } + None => { + sleep(Duration::from_millis(10)).await; + } + } + } +} diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index dbd66fe1f8..e29da9c51e 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -8,6 +8,7 @@ pub mod clickhouse; pub mod db; pub mod dendrite; +pub mod maghemite; pub mod poll; #[cfg(feature = "seed-gen")] pub mod seed; diff --git a/test-utils/src/dev/seed.rs b/test-utils/src/dev/seed.rs index 841ecd5f35..7a75d0bbd3 100644 --- a/test-utils/src/dev/seed.rs +++ b/test-utils/src/dev/seed.rs @@ -92,12 +92,14 @@ pub async fn ensure_seed_tarball_exists( ); } - // XXX: we aren't considering cross-user permissions for this file. Might be - // worth setting more restrictive permissions on it, or using a per-user - // cache dir. + // If possible, try for a per-user folder in the temp dir + // to avoid clashes on shared build environments. + let crdb_base = std::env::var("USER") + .map(|user| format!("crdb-base-{user}")) + .unwrap_or("crdb-base".into()); let base_seed_dir = Utf8PathBuf::from_path_buf(std::env::temp_dir()) .expect("Not a UTF-8 path") - .join("crdb-base"); + .join(crdb_base); std::fs::create_dir_all(&base_seed_dir).unwrap(); let mut desired_seed_tar = base_seed_dir.join(digest_unique_to_schema()); desired_seed_tar.set_extension("tar"); diff --git a/tools/build-global-zone-packages.sh b/tools/build-global-zone-packages.sh index 54af9d6327..fc1ab42ade 100755 --- a/tools/build-global-zone-packages.sh +++ b/tools/build-global-zone-packages.sh @@ -12,7 +12,7 @@ out_dir="$(readlink -f "${2:-"$tarball_src_dir"}")" # Make sure needed packages exist deps=( "$tarball_src_dir/omicron-sled-agent.tar" - "$tarball_src_dir/maghemite.tar" + "$tarball_src_dir/mg-ddm-gz.tar" "$tarball_src_dir/propolis-server.tar.gz" "$tarball_src_dir/overlay.tar.gz" ) @@ -46,7 +46,7 @@ cd - pkg_dir="$tmp_gz/root/opt/oxide/mg-ddm" mkdir -p "$pkg_dir" cd "$pkg_dir" -tar -xvfz "$tarball_src_dir/maghemite.tar" +tar -xvfz "$tarball_src_dir/mg-ddm-gz.tar" cd - # propolis should be bundled with this OS: Put the propolis-server zone image diff --git a/tools/build-host-image.sh b/tools/build-host-image.sh index d492e84b81..c194edb603 100755 --- a/tools/build-host-image.sh +++ b/tools/build-host-image.sh @@ -92,22 +92,6 @@ function main # Move to the helios checkout cd "$HELIOS_PATH" - # Create the "./helios-build" command, which lets us build images - gmake setup - - # Commands that "./helios-build" would ask us to run (either explicitly - # or implicitly, to avoid an error). - rc=0 - pfexec pkg install -q /system/zones/brand/omicron1/tools || rc=$? - case $rc in - # `man pkg` notes that exit code 4 means no changes were made because - # there is nothing to do; that's fine. Any other exit code is an error. - 0 | 4) ;; - *) exit $rc ;; - esac - - pfexec zfs create -p rpool/images/"$USER" - HELIOS_REPO=https://pkg.oxide.computer/helios/2/dev/ # Build an image name that includes the omicron and host OS hashes diff --git a/tools/build-trampoline-global-zone-packages.sh b/tools/build-trampoline-global-zone-packages.sh index 87013fb563..d8df0f8921 100755 --- a/tools/build-trampoline-global-zone-packages.sh +++ b/tools/build-trampoline-global-zone-packages.sh @@ -12,7 +12,7 @@ out_dir="$(readlink -f "${2:-$tarball_src_dir}")" # Make sure needed packages exist deps=( "$tarball_src_dir"/installinator.tar - "$tarball_src_dir"/maghemite.tar + "$tarball_src_dir"/mg-ddm-gz.tar ) for dep in "${deps[@]}"; do if [[ ! -e $dep ]]; then @@ -44,7 +44,7 @@ cd - pkg_dir="$tmp_trampoline/root/opt/oxide/mg-ddm" mkdir -p "$pkg_dir" cd "$pkg_dir" -tar -xvfz "$tarball_src_dir/maghemite.tar" +tar -xvfz "$tarball_src_dir/mg-ddm-gz.tar" cd - # Create the final output and we're done diff --git a/tools/ci_download_maghemite_mgd b/tools/ci_download_maghemite_mgd new file mode 100755 index 0000000000..eff680d7fd --- /dev/null +++ b/tools/ci_download_maghemite_mgd @@ -0,0 +1,168 @@ +#!/bin/bash + +# +# ci_download_maghemite_mgd: fetches the maghemite mgd binary tarball, unpacks +# it, and creates a copy called mgd, all in the current directory +# + +set -o pipefail +set -o xtrace +set -o errexit + +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +ARG0="$(basename "${BASH_SOURCE[0]}")" + +source "$SOURCE_DIR/maghemite_mgd_checksums" +source "$SOURCE_DIR/maghemite_mg_openapi_version" + +TARGET_DIR="out" +# Location where intermediate artifacts are downloaded / unpacked. +DOWNLOAD_DIR="$TARGET_DIR/downloads" +# Location where the final mgd directory should end up. +DEST_DIR="./$TARGET_DIR/mgd" +BIN_DIR="$DEST_DIR/root/opt/oxide/mgd/bin" + +ARTIFACT_URL="https://buildomat.eng.oxide.computer/public/file" + +REPO='oxidecomputer/maghemite' +PACKAGE_BASE_URL="$ARTIFACT_URL/$REPO/image/$COMMIT" + +function main +{ + # + # Process command-line arguments. We generally don't expect any, but + # we allow callers to specify a value to override OSTYPE, just for + # testing. + # + if [[ $# != 0 ]]; then + CIDL_OS="$1" + shift + else + CIDL_OS="$OSTYPE" + fi + + if [[ $# != 0 ]]; then + echo "unexpected arguments" >&2 + exit 2 + fi + + # Configure this program + configure_os "$CIDL_OS" + + CIDL_SHA256FUNC="do_sha256sum" + TARBALL_FILENAME="mgd.tar.gz" + PACKAGE_URL="$PACKAGE_BASE_URL/$TARBALL_FILENAME" + TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME" + + # Download the file. + echo "URL: $PACKAGE_URL" + echo "Local file: $TARBALL_FILE" + + mkdir -p "$DOWNLOAD_DIR" + mkdir -p "$DEST_DIR" + + fetch_and_verify + + do_untar "$TARBALL_FILE" + + do_assemble + + $SET_BINARIES +} + +function fail +{ + echo "$ARG0: $@" >&2 + exit 1 +} + +function configure_os +{ + echo "current directory: $PWD" + echo "configuring based on OS: \"$1\"" + case "$1" in + linux-gnu*) + SET_BINARIES="linux_binaries" + ;; + solaris*) + SET_BINARIES="" + ;; + *) + echo "WARNING: binaries for $1 are not published by maghemite" + echo "Dynamic routing apis will be unavailable" + SET_BINARIES="unsupported_os" + ;; + esac +} + +function do_download_curl +{ + curl --silent --show-error --fail --location --output "$2" "$1" +} + +function do_sha256sum +{ + sha256sum < "$1" | awk '{print $1}' +} + +function do_untar +{ + tar xzf "$1" -C "$DOWNLOAD_DIR" +} + +function do_assemble +{ + rm -r "$DEST_DIR" || true + mkdir "$DEST_DIR" + cp -r "$DOWNLOAD_DIR/root" "$DEST_DIR/root" +} + +function fetch_and_verify +{ + local DO_DOWNLOAD="true" + if [[ -f "$TARBALL_FILE" ]]; then + # If the file exists with a valid checksum, we can skip downloading. + calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \ + fail "failed to calculate sha256sum" + if [[ "$calculated_sha256" == "$CIDL_SHA256" ]]; then + DO_DOWNLOAD="false" + fi + fi + + if [ "$DO_DOWNLOAD" == "true" ]; then + echo "Downloading..." + do_download_curl "$PACKAGE_URL" "$TARBALL_FILE" || \ + fail "failed to download file" + + # Verify the sha256sum. + calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \ + fail "failed to calculate sha256sum" + if [[ "$calculated_sha256" != "$CIDL_SHA256" ]]; then + fail "sha256sum mismatch \ + (expected $CIDL_SHA256, found $calculated_sha256)" + fi + fi + +} + +function linux_binaries +{ + PACKAGE_BASE_URL="$ARTIFACT_URL/$REPO/linux/$COMMIT" + CIDL_SHA256="$MGD_LINUX_SHA256" + CIDL_SHA256FUNC="do_sha256sum" + TARBALL_FILENAME="mgd" + PACKAGE_URL="$PACKAGE_BASE_URL/$TARBALL_FILENAME" + TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME" + fetch_and_verify + chmod +x "$DOWNLOAD_DIR/mgd" + cp "$DOWNLOAD_DIR/mgd" "$BIN_DIR" +} + +function unsupported_os +{ + mkdir -p "$BIN_DIR" + echo "echo 'unsupported os' && exit 1" >> "$BIN_DIR/dpd" + chmod +x "$BIN_DIR/dpd" +} + +main "$@" diff --git a/tools/ci_download_maghemite_openapi b/tools/ci_download_maghemite_openapi index 37ff4f5547..db53f68d2c 100755 --- a/tools/ci_download_maghemite_openapi +++ b/tools/ci_download_maghemite_openapi @@ -15,10 +15,7 @@ TARGET_DIR="out" # Location where intermediate artifacts are downloaded / unpacked. DOWNLOAD_DIR="$TARGET_DIR/downloads" -source "$SOURCE_DIR/maghemite_openapi_version" -URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/openapi/$COMMIT/ddm-admin.json" -LOCAL_FILE="$DOWNLOAD_DIR/ddm-admin-$COMMIT.json" function main { @@ -83,4 +80,14 @@ function do_sha256sum $SHA < "$1" | awk '{print $1}' } +source "$SOURCE_DIR/maghemite_ddm_openapi_version" +URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/openapi/$COMMIT/ddm-admin.json" +LOCAL_FILE="$DOWNLOAD_DIR/ddm-admin-$COMMIT.json" + +main "$@" + +source "$SOURCE_DIR/maghemite_mg_openapi_version" +URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/openapi/$COMMIT/mg-admin.json" +LOCAL_FILE="$DOWNLOAD_DIR/mg-admin-$COMMIT.json" + main "$@" diff --git a/tools/ci_download_softnpu_machinery b/tools/ci_download_softnpu_machinery index 7975a310f0..cb5ea40210 100755 --- a/tools/ci_download_softnpu_machinery +++ b/tools/ci_download_softnpu_machinery @@ -15,7 +15,7 @@ OUT_DIR="out/npuzone" # Pinned commit for softnpu ASIC simulator SOFTNPU_REPO="softnpu" -SOFTNPU_COMMIT="eb27e6a00f1082c9faac7cf997e57d0609f7a309" +SOFTNPU_COMMIT="c1c42398c82b0220c8b5fa3bfba9c7a3bcaa0943" # This is the softnpu ASIC simulator echo "fetching npuzone" diff --git a/tools/console_version b/tools/console_version index 218aef576d..7e8d352efd 100644 --- a/tools/console_version +++ b/tools/console_version @@ -1,2 +1,2 @@ -COMMIT="3538c32a5189bd22df8f6a573399dacfbe81efaa" -SHA2="3289989f2cd6c71ea800e73231190455cc8e4e45ae9304293050b925a9fd9423" +COMMIT="bd65b9da7019ad812dd056e7fc182df2cf4ec128" +SHA2="e4d4f33996a6e89b972fac61737acb7f1dbd21943d1f6bef776d4ee9bcccd2b0" diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index 95c2aa63df..908cb752e9 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -44,6 +44,9 @@ function ensure_simulated_links { if [[ -z "$(get_vnic_name_if_exists "sc0_1")" ]]; then dladm create-vnic -t "sc0_1" -l "$PHYSICAL_LINK" -m a8:e1:de:01:70:1d + if [[ -v PROMISC_FILT_OFF ]]; then + dladm set-linkprop -p promisc-filtered=off sc0_1 + fi fi success "Vnic sc0_1 exists" } @@ -58,7 +61,8 @@ function ensure_softnpu_zone { out/npuzone/npuzone create sidecar \ --omicron-zone \ --ports sc0_0,tfportrear0_0 \ - --ports sc0_1,tfportqsfp0_0 + --ports sc0_1,tfportqsfp0_0 \ + --sidecar-lite-branch main } "$SOURCE_DIR"/scrimlet/softnpu-init.sh success "softnpu zone exists" diff --git a/tools/delete-reservoir.sh b/tools/delete-reservoir.sh new file mode 100755 index 0000000000..77e814f0c7 --- /dev/null +++ b/tools/delete-reservoir.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +size=`pfexec /usr/lib/rsrvrctl -q | grep Free | awk '{print $3}'` +let x=$size/1024 + +pfexec /usr/lib/rsrvrctl -r $x diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index b1f210a647..c91d1c2e98 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="7712104585266a2898da38c1345210ad26f9e71d" -SHA2="cb3f0cfbe6216d2441d34e0470252e0fb142332e47b33b65c24ef7368a694b6d" +COMMIT="343e3a572cc02efe3f8b68f9affd008623a33966" +SHA2="544ab42ccc7942d8ece9cdc80cd85d002bcf9d5646a291322bf2f79087ab6df0" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index 9538bc0d00..8fa98114fb 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="486b0b016c0df06947810b90f3a3dd40423f0ee6f255ed079dc8e5618c9a7281" -CIDL_SHA256_LINUX_DPD="af97aaf7e1046a5c651d316c384171df6387b4c54c8ae4a3ef498e532eaa5a4c" -CIDL_SHA256_LINUX_SWADM="909e400dcc9880720222c6dc3919404d83687f773f668160f66f38b51a81c188" +CIDL_SHA256_ILLUMOS="0808f331741e02d55e199847579dfd01f3658b21c7122cef8c3f9279f43dbab0" +CIDL_SHA256_LINUX_DPD="3e276dd553dd7cdb75c8ad023c2cd29b91485fafb94f27097a745b2b7ef5ecea" +CIDL_SHA256_LINUX_SWADM="645faf8a93bcae9814b2f116bccd66a54763332b56220e93b66316c853ce13d2" diff --git a/tools/dvt_dock_version b/tools/dvt_dock_version index 790bd3ec26..b52dded7d0 100644 --- a/tools/dvt_dock_version +++ b/tools/dvt_dock_version @@ -1 +1 @@ -COMMIT=7cbfa19bad077a3c42976357a317d18291533ba2 +COMMIT=9cb2b40cea90ad40f5c0d2c3da96d26913253e06 diff --git a/tools/hubris_checksums b/tools/hubris_checksums index e2b16d345e..1396af4d60 100644 --- a/tools/hubris_checksums +++ b/tools/hubris_checksums @@ -1,7 +1,7 @@ -204328b941deab8bfd3d6e34af96ef04489672fa3b0d5419b60456f9b052e789 build-gimlet-c-image-default-v1.0.2.zip -0ebaa9d98c3734420a482160a7a19dd09510ea3bdc573a090a97ec47137bd624 build-gimlet-d-image-default-v1.0.2.zip -39ec8fd0c946b744e59d9c1b89914f354c60f54e974398029d1dea9d31681f05 build-gimlet-e-image-default-v1.0.2.zip -fa5dc36a7a70eeb45d4c4b3b314ba54ee820b3d57ffc07defcc3ae07c142329c build-psc-b-image-default-v1.0.2.zip -4a9850100f8b5fcbbdd11d41ccd8d5037059697a9b65c1ba2dba48a6565ba9e7 build-psc-c-image-default-v1.0.2.zip -1bb870d7921c1731ec83dc38b8e3425703ec17efa614d75e92f07a551312f54b build-sidecar-b-image-default-v1.0.2.zip -6aed0e15e0025bb87a22ecea60d75fa71b54b83bea1e213b8cd5bdb02e7ccb2d build-sidecar-c-image-default-v1.0.2.zip +2df01d7dd17423588c99de4361694efdb6bd375e2f54db053320eeead3e07eda build-gimlet-c-image-default-v1.0.3.zip +8ac0eb6d7817825c6318feb8327f5758a33ccd2479512e3e2424f0eb8e290010 build-gimlet-d-image-default-v1.0.3.zip +eeeb72ec81a843fa1f5093096d1e4500aba6ce01c2d21040a2a10a092595d945 build-gimlet-e-image-default-v1.0.3.zip +de0d9028929322f6d5afc4cb52c198b3402c93a38aa15f9d378617ca1d1112c9 build-psc-b-image-default-v1.0.3.zip +11a6235d852bd75548f12d85b0913cb4ccb0aff3c38bf8a92510a2b9c14dad3c build-psc-c-image-default-v1.0.3.zip +3f863d46a462432f19d3fb5a293b8106da6e138de80271f869692bd29abd994b build-sidecar-b-image-default-v1.0.3.zip +2a9feac7f2da61b843d00edf2693c31c118f202c6cd889d1d1758ea1dd95dbca build-sidecar-c-image-default-v1.0.3.zip diff --git a/tools/hubris_version b/tools/hubris_version index 0f1729d791..b00c3286fe 100644 --- a/tools/hubris_version +++ b/tools/hubris_version @@ -1 +1 @@ -TAGS=(gimlet-v1.0.2 psc-v1.0.2 sidecar-v1.0.2) +TAGS=(gimlet-v1.0.3 psc-v1.0.3 sidecar-v1.0.3) diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index 62603ecac7..d3ecd8eaa8 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -197,6 +197,10 @@ retry ./tools/ci_download_dendrite_openapi # asic and running dendrite instance retry ./tools/ci_download_dendrite_stub +# Download mgd. This is required to run tests that invovle dynamic external +# routing +retry ./tools/ci_download_maghemite_mgd + # Download transceiver-control. This is used as the source for the # xcvradm binary which is bundled with the switch zone. retry ./tools/ci_download_transceiver_control diff --git a/tools/install_runner_prerequisites.sh b/tools/install_runner_prerequisites.sh index 7ece993bc9..42347f518d 100755 --- a/tools/install_runner_prerequisites.sh +++ b/tools/install_runner_prerequisites.sh @@ -105,6 +105,7 @@ function install_packages { 'pkg-config' 'brand/omicron1/tools' 'library/libxmlsec1' + 'chrony' ) # Install/update the set of packages. @@ -119,13 +120,15 @@ function install_packages { exit "$rc" fi + pfexec svcadm enable chrony + pkg list -v "${packages[@]}" elif [[ "${HOST_OS}" == "Linux" ]]; then packages=( 'ca-certificates' 'libpq5' 'libsqlite3-0' - 'libssl1.1' + 'libssl3' 'libxmlsec1-openssl' ) sudo apt-get update diff --git a/tools/maghemite_openapi_version b/tools/maghemite_ddm_openapi_version similarity index 59% rename from tools/maghemite_openapi_version rename to tools/maghemite_ddm_openapi_version index 8f84b30cb1..89c3e46164 100644 --- a/tools/maghemite_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="12703675393459e74139f8140e0b3c4c4f129d5d" +COMMIT="d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version new file mode 100644 index 0000000000..a7e18285ae --- /dev/null +++ b/tools/maghemite_mg_openapi_version @@ -0,0 +1,2 @@ +COMMIT="d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" +SHA2="d0f7611e5ecd049b0f83bcfa843942401f155a0be36d9a2dfd73b8341d5f816e" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums new file mode 100644 index 0000000000..e65e1fc0a2 --- /dev/null +++ b/tools/maghemite_mgd_checksums @@ -0,0 +1,2 @@ +CIDL_SHA256="452dfb3491e1b6d4df6be1cb689921f59623aed082e47606a78c0f44d918f66a" +MGD_LINUX_SHA256="d4c48eb6374c0cc7812b7af2c0ac92acdcbc91b7718a9ce64d069da00ae5ae73" diff --git a/tools/opte_version b/tools/opte_version index 2dbaeb7154..0a79a6aba9 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.23.181 +0.25.183 diff --git a/tools/update_maghemite.sh b/tools/update_maghemite.sh index a4a9b1291e..eebece1aa5 100755 --- a/tools/update_maghemite.sh +++ b/tools/update_maghemite.sh @@ -15,8 +15,9 @@ function usage { } PACKAGES=( - "maghemite" + "mg-ddm-gz" "mg-ddm" + "mgd" ) REPO="oxidecomputer/maghemite" @@ -26,13 +27,14 @@ REPO="oxidecomputer/maghemite" function update_openapi { TARGET_COMMIT="$1" DRY_RUN="$2" - SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "ddm-admin.json" "openapi") + DAEMON="$3" + SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "${DAEMON}-admin.json" "openapi") OUTPUT=$(printf "COMMIT=\"%s\"\nSHA2=\"%s\"\n" "$TARGET_COMMIT" "$SHA") if [ -n "$DRY_RUN" ]; then OPENAPI_PATH="/dev/null" else - OPENAPI_PATH="$SOURCE_DIR/maghemite_openapi_version" + OPENAPI_PATH="$SOURCE_DIR/maghemite_${DAEMON}_openapi_version" fi echo "Updating Maghemite OpenAPI from: $TARGET_COMMIT" set -x @@ -40,6 +42,27 @@ function update_openapi { set +x } +function update_mgd { + TARGET_COMMIT="$1" + DRY_RUN="$2" + DAEMON="$3" + SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "image") + OUTPUT=$(printf "CIDL_SHA256=\"%s\"\n" "$SHA") + + SHA_LINUX=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "linux") + OUTPUT_LINUX=$(printf "MGD_LINUX_SHA256=\"%s\"\n" "$SHA_LINUX") + + if [ -n "$DRY_RUN" ]; then + MGD_PATH="/dev/null" + else + MGD_PATH="$SOURCE_DIR/maghemite_mgd_checksums" + fi + echo "Updating Maghemite mgd from: $TARGET_COMMIT" + set -x + echo "$OUTPUT\n$OUTPUT_LINUX" > $MGD_PATH + set +x +} + function main { TARGET_COMMIT="" DRY_RUN="" @@ -60,7 +83,9 @@ function main { TARGET_COMMIT=$(get_latest_commit_from_gh "$REPO" "$TARGET_COMMIT") install_toml2json do_update_packages "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${PACKAGES[@]}" - update_openapi "$TARGET_COMMIT" "$DRY_RUN" + update_openapi "$TARGET_COMMIT" "$DRY_RUN" ddm + update_openapi "$TARGET_COMMIT" "$DRY_RUN" mg + update_mgd "$TARGET_COMMIT" "$DRY_RUN" do_update_packages "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${PACKAGES[@]}" } diff --git a/update-engine/src/buffer.rs b/update-engine/src/buffer.rs index d1028ff8cc..3e7db63cb9 100644 --- a/update-engine/src/buffer.rs +++ b/update-engine/src/buffer.rs @@ -240,8 +240,13 @@ impl EventStore { // index. let root_event_index = RootEventIndex(event.event_index); - let actions = - self.recurse_for_step_event(&event, 0, None, root_event_index); + let actions = self.recurse_for_step_event( + &event, + 0, + None, + root_event_index, + event.total_elapsed, + ); if let Some(new_execution) = actions.new_execution { if new_execution.nest_level == 0 { self.root_execution_id = Some(new_execution.execution_id); @@ -312,6 +317,7 @@ impl EventStore { nest_level: usize, parent_sort_key: Option<&StepSortKey>, root_event_index: RootEventIndex, + root_total_elapsed: Duration, ) -> RecurseActions { let mut new_execution = None; let (step_key, progress_key) = match &event.kind { @@ -365,6 +371,8 @@ impl EventStore { let info = CompletionInfo { attempt: *attempt, outcome, + root_total_elapsed, + leaf_total_elapsed: event.total_elapsed, step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; @@ -405,6 +413,8 @@ impl EventStore { let info = CompletionInfo { attempt: *last_attempt, outcome, + root_total_elapsed, + leaf_total_elapsed: event.total_elapsed, step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; @@ -432,6 +442,8 @@ impl EventStore { total_attempts: *total_attempts, message: message.clone(), causes: causes.clone(), + root_total_elapsed, + leaf_total_elapsed: event.total_elapsed, step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; @@ -456,6 +468,8 @@ impl EventStore { let info = AbortInfo { attempt: *attempt, message: message.clone(), + root_total_elapsed, + leaf_total_elapsed: event.total_elapsed, step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; @@ -481,6 +495,7 @@ impl EventStore { nest_level + 1, parent_sort_key.as_ref(), root_event_index, + root_total_elapsed, ); if let Some(nested_new_execution) = &actions.new_execution { // Add an edge from the parent node to the new execution's root node. @@ -1164,6 +1179,8 @@ impl StepStatus { pub struct CompletionInfo { pub attempt: usize, pub outcome: StepOutcome, + pub root_total_elapsed: Duration, + pub leaf_total_elapsed: Duration, pub step_elapsed: Duration, pub attempt_elapsed: Duration, } @@ -1179,11 +1196,23 @@ pub enum FailureReason { }, } +impl FailureReason { + /// Returns the [`FailureInfo`] if present. + pub fn info(&self) -> Option<&FailureInfo> { + match self { + Self::StepFailed(info) => Some(info), + Self::ParentFailed { .. } => None, + } + } +} + #[derive(Clone, Debug)] pub struct FailureInfo { pub total_attempts: usize, pub message: String, pub causes: Vec, + pub root_total_elapsed: Duration, + pub leaf_total_elapsed: Duration, pub step_elapsed: Duration, pub attempt_elapsed: Duration, } @@ -1199,6 +1228,16 @@ pub enum AbortReason { }, } +impl AbortReason { + /// Returns the [`AbortInfo`] if present. + pub fn info(&self) -> Option<&AbortInfo> { + match self { + Self::StepAborted(info) => Some(info), + Self::ParentAborted { .. } => None, + } + } +} + #[derive(Clone, Debug)] pub enum WillNotBeRunReason { /// A preceding step failed. @@ -1230,6 +1269,13 @@ pub enum WillNotBeRunReason { pub struct AbortInfo { pub attempt: usize, pub message: String, + + /// The total elapsed time as reported by the root event. + pub root_total_elapsed: Duration, + + /// The total elapsed time as reported by the leaf execution event, for + /// nested events. + pub leaf_total_elapsed: Duration, pub step_elapsed: Duration, pub attempt_elapsed: Duration, } @@ -1261,14 +1307,61 @@ impl ExecutionSummary { StepStatus::Running { .. } => { execution_status = ExecutionStatus::Running { step_key }; } - StepStatus::Completed { .. } => { - execution_status = ExecutionStatus::Completed { step_key }; + StepStatus::Completed { info } => { + let (root_total_elapsed, leaf_total_elapsed) = match info { + Some(info) => ( + Some(info.root_total_elapsed), + Some(info.leaf_total_elapsed), + ), + None => (None, None), + }; + + let terminal_status = ExecutionTerminalInfo { + kind: TerminalKind::Completed, + root_total_elapsed, + leaf_total_elapsed, + step_key, + }; + execution_status = + ExecutionStatus::Terminal(terminal_status); } - StepStatus::Failed { .. } => { - execution_status = ExecutionStatus::Failed { step_key }; + StepStatus::Failed { reason } => { + let (root_total_elapsed, leaf_total_elapsed) = + match reason.info() { + Some(info) => ( + Some(info.root_total_elapsed), + Some(info.leaf_total_elapsed), + ), + None => (None, None), + }; + + let terminal_status = ExecutionTerminalInfo { + kind: TerminalKind::Failed, + root_total_elapsed, + leaf_total_elapsed, + step_key, + }; + execution_status = + ExecutionStatus::Terminal(terminal_status); } - StepStatus::Aborted { .. } => { - execution_status = ExecutionStatus::Aborted { step_key }; + StepStatus::Aborted { reason, .. } => { + let (root_total_elapsed, leaf_total_elapsed) = + match reason.info() { + Some(info) => ( + Some(info.root_total_elapsed), + Some(info.leaf_total_elapsed), + ), + None => (None, None), + }; + + let terminal_status = ExecutionTerminalInfo { + kind: TerminalKind::Aborted, + root_total_elapsed, + leaf_total_elapsed, + step_key, + }; + execution_status = + ExecutionStatus::Terminal(terminal_status); } StepStatus::WillNotBeRun { .. } => { // Ignore steps that will not be run -- a prior step failed. @@ -1306,7 +1399,7 @@ impl StepSortKey { /// Status about a single execution ID. /// /// Part of [`ExecutionSummary`]. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum ExecutionStatus { /// This execution has not been started yet. NotStarted, @@ -1319,27 +1412,50 @@ pub enum ExecutionStatus { step_key: StepKey, }, - /// This execution completed running. - Completed { - /// The last step that completed. - step_key: StepKey, - }, + /// Execution has finished. + Terminal(ExecutionTerminalInfo), +} - /// This execution failed. - Failed { - /// The step key that failed. - /// - /// Use [`EventBuffer::get`] to get more information about this step. - step_key: StepKey, - }, +/// Terminal status about a single execution ID. +/// +/// Part of [`ExecutionStatus`]. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionTerminalInfo { + /// The way in which this execution reached a terminal state. + pub kind: TerminalKind, + + /// Total elapsed time (root) for this execution. + /// + /// The total elapsed time may not be available if execution was interrupted + /// and we inferred that it was terminated. + pub root_total_elapsed: Option, + + /// Total elapsed time (leaf) for this execution. + /// + /// The total elapsed time may not be available if execution was interrupted + /// and we inferred that it was terminated. + pub leaf_total_elapsed: Option, + /// The step key that was running when this execution was terminated. + /// + /// * For completed executions, this is the last step that completed. + /// * For failed or aborted executions, this is the step that failed. + /// * For aborted executions, this is the step that was running when the + /// abort happened. + pub step_key: StepKey, +} + +/// The way in which an execution was terminated. +/// +/// Part of [`ExecutionStatus`]. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TerminalKind { + /// This execution completed running. + Completed, + /// This execution failed. + Failed, /// This execution was aborted. - Aborted { - /// The step that was running when the abort happened. - /// - /// Use [`EventBuffer::get`] to get more information about this step. - step_key: StepKey, - }, + Aborted, } /// Keys for the event tree. @@ -1925,8 +2041,9 @@ mod tests { if is_last_event { ensure!( matches!( - summary[&root_execution_id].execution_status, - ExecutionStatus::Completed { .. }, + &summary[&root_execution_id].execution_status, + ExecutionStatus::Terminal(info) + if info.kind == TerminalKind::Completed ), "this is the last event so ExecutionStatus must be completed" ); @@ -1941,8 +2058,9 @@ mod tests { .expect("this is the first nested engine"); ensure!( matches!( - nested_summary.execution_status, - ExecutionStatus::Failed { .. }, + &nested_summary.execution_status, + ExecutionStatus::Terminal(info) + if info.kind == TerminalKind::Failed ), "for this engine, the ExecutionStatus must be failed" ); @@ -1952,8 +2070,9 @@ mod tests { .expect("this is the second nested engine"); ensure!( matches!( - nested_summary.execution_status, - ExecutionStatus::Completed { .. }, + &nested_summary.execution_status, + ExecutionStatus::Terminal(info) + if info.kind == TerminalKind::Completed ), "for this engine, the ExecutionStatus must be succeeded" ); diff --git a/update-engine/src/context.rs b/update-engine/src/context.rs index d232d931a2..cd85687cf9 100644 --- a/update-engine/src/context.rs +++ b/update-engine/src/context.rs @@ -223,7 +223,7 @@ impl StepContext { } } -/// Tracker for [`StepContext::add_nested_report`]. +/// Tracker for [`StepContext::send_nested_report`]. /// /// Nested event reports might contain events already seen in prior runs: /// `NestedEventBuffer` deduplicates those events such that only deltas are sent diff --git a/wicket-common/src/update_events.rs b/wicket-common/src/update_events.rs index 3dd984d07f..ac840f83ad 100644 --- a/wicket-common/src/update_events.rs +++ b/wicket-common/src/update_events.rs @@ -159,6 +159,21 @@ pub enum UpdateTerminalError { #[source] error: gateway_client::Error, }, + #[error("getting RoT CMPA failed")] + GetRotCmpaFailed { + #[source] + error: anyhow::Error, + }, + #[error("getting RoT CFPA failed")] + GetRotCfpaFailed { + #[source] + error: anyhow::Error, + }, + #[error("failed to find correctly-singed RoT image")] + FailedFindingSignedRotImage { + #[source] + error: anyhow::Error, + }, #[error("getting SP caboose failed")] GetSpCabooseFailed { #[source] diff --git a/wicket/src/rack_setup/config_template.toml b/wicket/src/rack_setup/config_template.toml index 4b193a0c29..617b61fadc 100644 --- a/wicket/src/rack_setup/config_template.toml +++ b/wicket/src/rack_setup/config_template.toml @@ -40,18 +40,24 @@ bootstrap_sleds = [] # TODO: docs on network config [rack_network_config] +rack_subnet = "" infra_ip_first = "" infra_ip_last = "" -[[rack_network_config.uplinks]] +[[rack_network_config.ports]] +# Routes associated with this port. +# { nexthop = "1.2.3.4", destination = "0.0.0.0/0" } +routes = [] + +# Addresses associated with this port. +# "1.2.3.4/24" +addresses = [] + # Either `switch0` or `switch1`, matching the hardware. switch = "" -# IP address this uplink should use as its gateway. -gateway_ip = "" - # qsfp0, qsfp1, ... -uplink_port = "" +port = "" # `speed40_g`, `speed100_g`, ... uplink_port_speed = "" @@ -59,8 +65,14 @@ uplink_port_speed = "" # `none`, `firecode`, or `rs` uplink_port_fec = "" -# IP address and prefix for this uplink; e.g., `192.168.100.100/16` -uplink_cidr = "" +# A list of bgp peers +# { addr = "1.7.0.1", asn = 47, port = "qsfp0" } +bgp_peers = [] + +# Optional BGP configuration. Remove this section if not needed. +[[rack_network_config.bgp]] +# The autonomous system numer +asn = 0 -# VLAN ID for this uplink; omit if no VLAN ID is needed -uplink_vid = 1234 +# Prefixes to originate e.g., ["10.0.0.0/16"] +originate = [] diff --git a/wicket/src/rack_setup/config_toml.rs b/wicket/src/rack_setup/config_toml.rs index 5f0bb9e876..e087c9aa7c 100644 --- a/wicket/src/rack_setup/config_toml.rs +++ b/wicket/src/rack_setup/config_toml.rs @@ -18,7 +18,7 @@ use toml_edit::Value; use wicketd_client::types::BootstrapSledDescription; use wicketd_client::types::CurrentRssUserConfigInsensitive; use wicketd_client::types::IpRange; -use wicketd_client::types::RackNetworkConfig; +use wicketd_client::types::RackNetworkConfigV1; use wicketd_client::types::SpType; static TEMPLATE: &str = include_str!("config_template.toml"); @@ -176,7 +176,7 @@ fn build_sleds_array(sleds: &[BootstrapSledDescription]) -> Array { fn populate_network_table( table: &mut Table, - config: Option<&RackNetworkConfig>, + config: Option<&RackNetworkConfigV1>, ) { // Helper function to serialize enums into their appropriate string // representations. @@ -195,6 +195,7 @@ fn populate_network_table( }; for (property, value) in [ + ("rack_subnet", config.rack_subnet.to_string()), ("infra_ip_first", config.infra_ip_first.to_string()), ("infra_ip_last", config.infra_ip_last.to_string()), ] { @@ -202,20 +203,17 @@ fn populate_network_table( Value::String(Formatted::new(value)); } - // If `config.uplinks` is empty, we'll leave the template uplinks in place; - // otherwise, replace it with the user's uplinks. - if !config.uplinks.is_empty() { - *table.get_mut("uplinks").unwrap().as_array_of_tables_mut().unwrap() = + if !config.ports.is_empty() { + *table.get_mut("ports").unwrap().as_array_of_tables_mut().unwrap() = config - .uplinks + .ports .iter() .map(|cfg| { let mut uplink = Table::new(); - let mut last_key = None; + let mut _last_key = None; for (property, value) in [ ("switch", cfg.switch.to_string()), - ("gateway_ip", cfg.gateway_ip.to_string()), - ("uplink_port", cfg.uplink_port.to_string()), + ("port", cfg.port.to_string()), ( "uplink_port_speed", enum_to_toml_string(&cfg.uplink_port_speed), @@ -224,63 +222,121 @@ fn populate_network_table( "uplink_port_fec", enum_to_toml_string(&cfg.uplink_port_fec), ), - ("uplink_cidr", cfg.uplink_cidr.to_string()), ] { uplink.insert( property, Item::Value(Value::String(Formatted::new(value))), ); - last_key = Some(property); + _last_key = Some(property); } - if let Some(uplink_vid) = cfg.uplink_vid { - uplink.insert( - "uplink_vid", - Item::Value(Value::Integer(Formatted::new( - i64::from(uplink_vid), - ))), + let mut routes = Array::new(); + for r in &cfg.routes { + let mut route = InlineTable::new(); + route.insert( + "nexthop", + Value::String(Formatted::new( + r.nexthop.to_string(), + )), + ); + route.insert( + "destination", + Value::String(Formatted::new( + r.destination.to_string(), + )), ); - } else { - // Unwraps: We know `last_key` is `Some(_)`, because we - // set it in every iteration of the loop above, and we - // know it's present in `uplink` because we set it to - // the `property` we just inserted. - let last = uplink.get_mut(last_key.unwrap()).unwrap(); - - // Every item we insert is an `Item::Value`, so we can - // unwrap this conversion. - last.as_value_mut() - .unwrap() - .decor_mut() - .set_suffix("\n# uplink_vid ="); + routes.push(Value::InlineTable(route)); } + uplink.insert("routes", Item::Value(Value::Array(routes))); + let mut addresses = Array::new(); + for a in &cfg.addresses { + addresses + .push(Value::String(Formatted::new(a.to_string()))) + } + uplink.insert( + "addresses", + Item::Value(Value::Array(addresses)), + ); + + let mut peers = Array::new(); + for p in &cfg.bgp_peers { + let mut peer = InlineTable::new(); + peer.insert( + "addr", + Value::String(Formatted::new(p.addr.to_string())), + ); + peer.insert( + "asn", + Value::Integer(Formatted::new(p.asn as i64)), + ); + peer.insert( + "port", + Value::String(Formatted::new(p.port.to_string())), + ); + peers.push(Value::InlineTable(peer)); + } + uplink + .insert("bgp_peers", Item::Value(Value::Array(peers))); uplink }) .collect(); } + if !config.bgp.is_empty() { + *table.get_mut("bgp").unwrap().as_array_of_tables_mut().unwrap() = + config + .bgp + .iter() + .map(|cfg| { + let mut bgp = Table::new(); + bgp.insert( + "asn", + Item::Value(Value::Integer(Formatted::new( + cfg.asn as i64, + ))), + ); + + let mut originate = Array::new(); + for o in &cfg.originate { + originate + .push(Value::String(Formatted::new(o.to_string()))); + } + bgp.insert( + "originate", + Item::Value(Value::Array(originate)), + ); + bgp + }) + .collect(); + } } #[cfg(test)] mod tests { use super::*; - use omicron_common::api::internal::shared::RackNetworkConfig as InternalRackNetworkConfig; + use omicron_common::api::internal::shared::RackNetworkConfigV1 as InternalRackNetworkConfig; use std::net::Ipv6Addr; use wicket_common::rack_setup::PutRssUserConfigInsensitive; use wicketd_client::types::Baseboard; + use wicketd_client::types::BgpConfig; + use wicketd_client::types::BgpPeerConfig; + use wicketd_client::types::PortConfigV1; use wicketd_client::types::PortFec; use wicketd_client::types::PortSpeed; + use wicketd_client::types::RouteConfig; use wicketd_client::types::SpIdentifier; use wicketd_client::types::SwitchLocation; - use wicketd_client::types::UplinkConfig; fn put_config_from_current_config( value: CurrentRssUserConfigInsensitive, ) -> PutRssUserConfigInsensitive { + use omicron_common::api::internal::shared::BgpConfig as InternalBgpConfig; + use omicron_common::api::internal::shared::BgpPeerConfig as InternalBgpPeerConfig; + use omicron_common::api::internal::shared::PortConfigV1 as InternalPortConfig; use omicron_common::api::internal::shared::PortFec as InternalPortFec; use omicron_common::api::internal::shared::PortSpeed as InternalPortSpeed; + use omicron_common::api::internal::shared::RouteConfig as InternalRouteConfig; use omicron_common::api::internal::shared::SwitchLocation as InternalSwitchLocation; - use omicron_common::api::internal::shared::UplinkConfig as InternalUplinkConfig; let rnc = value.rack_network_config.unwrap(); @@ -310,14 +366,32 @@ mod tests { external_dns_ips: value.external_dns_ips, ntp_servers: value.ntp_servers, rack_network_config: InternalRackNetworkConfig { + rack_subnet: rnc.rack_subnet, infra_ip_first: rnc.infra_ip_first, infra_ip_last: rnc.infra_ip_last, - uplinks: rnc - .uplinks + ports: rnc + .ports .iter() - .map(|config| InternalUplinkConfig { - gateway_ip: config.gateway_ip, - uplink_port: config.uplink_port.clone(), + .map(|config| InternalPortConfig { + routes: config + .routes + .iter() + .map(|r| InternalRouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), + bgp_peers: config + .bgp_peers + .iter() + .map(|p| InternalBgpPeerConfig { + asn: p.asn, + port: p.port.clone(), + addr: p.addr, + }) + .collect(), + port: config.port.clone(), uplink_port_speed: match config.uplink_port_speed { PortSpeed::Speed0G => InternalPortSpeed::Speed0G, PortSpeed::Speed1G => InternalPortSpeed::Speed1G, @@ -340,8 +414,6 @@ mod tests { PortFec::None => InternalPortFec::None, PortFec::Rs => InternalPortFec::Rs, }, - uplink_cidr: config.uplink_cidr, - uplink_vid: config.uplink_vid, switch: match config.switch { SwitchLocation::Switch0 => { InternalSwitchLocation::Switch0 @@ -352,6 +424,14 @@ mod tests { }, }) .collect(), + bgp: rnc + .bgp + .iter() + .map(|config| InternalBgpConfig { + asn: config.asn, + originate: config.originate.clone(), + }) + .collect(), }, } } @@ -392,18 +472,30 @@ mod tests { )], external_dns_ips: vec!["10.0.0.1".parse().unwrap()], ntp_servers: vec!["ntp1.com".into(), "ntp2.com".into()], - rack_network_config: Some(RackNetworkConfig { + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: "fd00:1122:3344:01::/56".parse().unwrap(), infra_ip_first: "172.30.0.1".parse().unwrap(), infra_ip_last: "172.30.0.10".parse().unwrap(), - uplinks: vec![UplinkConfig { - gateway_ip: "172.30.0.10".parse().unwrap(), - uplink_cidr: "172.30.0.1/24".parse().unwrap(), + ports: vec![PortConfigV1 { + addresses: vec!["172.30.0.1/24".parse().unwrap()], + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: "172.30.0.10".parse().unwrap(), + }], + bgp_peers: vec![BgpPeerConfig { + asn: 47, + addr: "10.2.3.4".parse().unwrap(), + port: "port0".into(), + }], uplink_port_speed: PortSpeed::Speed400G, uplink_port_fec: PortFec::Firecode, - uplink_port: "port0".into(), - uplink_vid: None, + port: "port0".into(), switch: SwitchLocation::Switch0, }], + bgp: vec![BgpConfig { + asn: 47, + originate: vec!["10.0.0.0/16".parse().unwrap()], + }], }), }; let template = TomlTemplate::populate(&config).to_string(); diff --git a/wicket/src/state/inventory.rs b/wicket/src/state/inventory.rs index 4b439c1414..23a0e244cf 100644 --- a/wicket/src/state/inventory.rs +++ b/wicket/src/state/inventory.rs @@ -6,7 +6,6 @@ use anyhow::anyhow; use once_cell::sync::Lazy; -use ratatui::text::Text; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fmt::Display; @@ -185,7 +184,7 @@ impl Component { } } -// The component type and its slot. +/// The component type and its slot. #[derive( Debug, Clone, @@ -205,27 +204,24 @@ pub enum ComponentId { } impl ComponentId { - pub fn name(&self) -> String { - self.to_string() + pub fn to_string_uppercase(&self) -> String { + let mut s = self.to_string(); + s.make_ascii_uppercase(); + s } } +/// Prints the component type in standard case. impl Display for ComponentId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - ComponentId::Sled(i) => write!(f, "SLED {}", i), - ComponentId::Switch(i) => write!(f, "SWITCH {}", i), + ComponentId::Sled(i) => write!(f, "sled {}", i), + ComponentId::Switch(i) => write!(f, "switch {}", i), ComponentId::Psc(i) => write!(f, "PSC {}", i), } } } -impl From for Text<'_> { - fn from(value: ComponentId) -> Self { - value.to_string().into() - } -} - pub struct ParsableComponentId<'a> { pub sp_type: &'a str, pub i: &'a str, @@ -269,3 +265,15 @@ impl PowerState { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn component_id_display() { + assert_eq!(ComponentId::Sled(0).to_string(), "sled 0"); + assert_eq!(ComponentId::Switch(1).to_string(), "switch 1"); + assert_eq!(ComponentId::Psc(2).to_string(), "PSC 2"); + } +} diff --git a/wicket/src/ui/main.rs b/wicket/src/ui/main.rs index 42cc6bf587..58ea6c1771 100644 --- a/wicket/src/ui/main.rs +++ b/wicket/src/ui/main.rs @@ -23,7 +23,7 @@ use wicketd_client::types::GetLocationResponse; /// This structure allows us to maintain similar styling and navigation /// throughout wicket with a minimum of code. /// -/// Specific functionality is put inside [`Pane`]s, which can be customized +/// Specific functionality is put inside Panes, which can be customized /// as needed. pub struct MainScreen { #[allow(unused)] diff --git a/wicket/src/ui/panes/overview.rs b/wicket/src/ui/panes/overview.rs index 3e0b317df9..e8cf50bb32 100644 --- a/wicket/src/ui/panes/overview.rs +++ b/wicket/src/ui/panes/overview.rs @@ -343,7 +343,7 @@ impl Control for InventoryView { let title_bar = Paragraph::new(Line::from(vec![ Span::styled("OXIDE RACK / ", border_style), Span::styled( - state.rack_state.selected.to_string(), + state.rack_state.selected.to_string_uppercase(), component_style, ), ])) diff --git a/wicket/src/ui/panes/rack_setup.rs b/wicket/src/ui/panes/rack_setup.rs index 212ddff4da..086d01ce9d 100644 --- a/wicket/src/ui/panes/rack_setup.rs +++ b/wicket/src/ui/panes/rack_setup.rs @@ -695,56 +695,61 @@ fn rss_config_text<'a>( }; if let Some(cfg) = insensitive.rack_network_config.as_ref() { - for (i, uplink) in cfg.uplinks.iter().enumerate() { + for (i, uplink) in cfg.ports.iter().enumerate() { let mut items = vec![ vec![ - Span::styled(" • Switch : ", label_style), + Span::styled(" • Switch : ", label_style), Span::styled(uplink.switch.to_string(), ok_style), ], vec![ - Span::styled(" • Gateway IP : ", label_style), - Span::styled(uplink.gateway_ip.to_string(), ok_style), - ], - vec![ - Span::styled(" • Uplink CIDR : ", label_style), - Span::styled(uplink.uplink_cidr.to_string(), ok_style), - ], - vec![ - Span::styled(" • Uplink port : ", label_style), - Span::styled(uplink.uplink_port.clone(), ok_style), - ], - vec![ - Span::styled(" • Uplink port speed: ", label_style), + Span::styled(" • Speed : ", label_style), Span::styled( uplink.uplink_port_speed.to_string(), ok_style, ), ], vec![ - Span::styled(" • Uplink port FEC : ", label_style), + Span::styled(" • FEC : ", label_style), Span::styled(uplink.uplink_port_fec.to_string(), ok_style), ], ]; - if let Some(uplink_vid) = uplink.uplink_vid { - items.push(vec![ - Span::styled(" • Uplink VLAN id : ", label_style), - Span::styled(uplink_vid.to_string(), ok_style), - ]); - } else { - items.push(vec![ - Span::styled(" • Uplink VLAN id : ", label_style), - Span::styled("none", ok_style), - ]); - } + + let routes = uplink.routes.iter().map(|r| { + vec![ + Span::styled(" • Route : ", label_style), + Span::styled( + format!("{} -> {}", r.destination, r.nexthop), + ok_style, + ), + ] + }); + + let addresses = uplink.addresses.iter().map(|a| { + vec![ + Span::styled(" • Address : ", label_style), + Span::styled(a.to_string(), ok_style), + ] + }); + + let peers = uplink.bgp_peers.iter().map(|p| { + vec![ + Span::styled(" • BGP peer : ", label_style), + Span::styled(format!("{} ASN={}", p.addr, p.asn), ok_style), + ] + }); + + items.extend(routes); + items.extend(addresses); + items.extend(peers); append_list( &mut spans, - Cow::from(format!("Uplink {}: ", i + 1)), + Cow::from(format!("Port {}: ", i + 1)), items, ); } } else { - append_list(&mut spans, "Uplinks: ".into(), vec![]); + append_list(&mut spans, "Ports: ".into(), vec![]); } append_list( diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs index da6f10cf88..2819b3ddda 100644 --- a/wicket/src/ui/panes/update.rs +++ b/wicket/src/ui/panes/update.rs @@ -29,7 +29,8 @@ use ratatui::widgets::{ use slog::{info, o, Logger}; use tui_tree_widget::{Tree, TreeItem, TreeState}; use update_engine::{ - AbortReason, ExecutionStatus, FailureReason, StepKey, WillNotBeRunReason, + AbortReason, ExecutionStatus, FailureReason, StepKey, TerminalKind, + WillNotBeRunReason, }; use wicket_common::update_events::{ EventBuffer, EventReport, ProgressEvent, StepOutcome, StepStatus, @@ -180,7 +181,7 @@ impl UpdatePane { tree_state, items: ALL_COMPONENT_IDS .iter() - .map(|id| TreeItem::new(*id, vec![])) + .map(|id| TreeItem::new(id.to_string_uppercase(), vec![])) .collect(), help: vec![ ("Expand", ""), @@ -531,7 +532,10 @@ impl UpdatePane { ) { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("START UPDATE: {}", state.rack_state.selected), + format!( + "START UPDATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body: Text::from(vec![Line::from(vec![Span::styled( @@ -561,7 +565,10 @@ impl UpdatePane { ) { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("START UPDATE: {}", state.rack_state.selected), + format!( + "START UPDATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body: Text::from(vec![Line::from(vec![Span::styled( @@ -594,7 +601,10 @@ impl UpdatePane { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("START UPDATE FAILED: {}", state.rack_state.selected), + format!( + "START UPDATE FAILED: {}", + state.rack_state.selected.to_string_uppercase() + ), style::failed_update(), )]), body, @@ -635,7 +645,10 @@ impl UpdatePane { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("ABORT UPDATE: {}", state.rack_state.selected), + format!( + "ABORT UPDATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body, @@ -662,7 +675,10 @@ impl UpdatePane { ) { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("ABORT UPDATE: {}", state.rack_state.selected), + format!( + "ABORT UPDATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body: Text::from(vec![Line::from(vec![Span::styled( @@ -695,7 +711,10 @@ impl UpdatePane { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("ABORT UPDATE FAILED: {}", state.rack_state.selected), + format!( + "ABORT UPDATE FAILED: {}", + state.rack_state.selected.to_string_uppercase() + ), style::failed_update(), )]), body, @@ -721,7 +740,10 @@ impl UpdatePane { ) { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("CLEAR UPDATE STATE: {}", state.rack_state.selected), + format!( + "CLEAR UPDATE STATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body: Text::from(vec![Line::from(vec![Span::styled( @@ -756,7 +778,7 @@ impl UpdatePane { header: Line::from(vec![Span::styled( format!( "CLEAR UPDATE STATE FAILED: {}", - state.rack_state.selected + state.rack_state.selected.to_string_uppercase() ), style::failed_update(), )]), @@ -830,7 +852,7 @@ impl UpdatePane { }) }) .collect(); - TreeItem::new(*id, children) + TreeItem::new(id.to_string_uppercase(), children) }) .collect(); } @@ -988,9 +1010,7 @@ impl UpdatePane { } ExecutionStatus::NotStarted - | ExecutionStatus::Completed { .. } - | ExecutionStatus::Failed { .. } - | ExecutionStatus::Aborted { .. } => None, + | ExecutionStatus::Terminal(_) => None, } } else { None @@ -1020,9 +1040,7 @@ impl UpdatePane { associated with it", ); match summary.execution_status { - ExecutionStatus::Completed { .. } - | ExecutionStatus::Failed { .. } - | ExecutionStatus::Aborted { .. } => { + ExecutionStatus::Terminal(_) => { // If execution has reached a terminal // state, we can clear it. self.popup = @@ -1107,7 +1125,11 @@ impl UpdatePane { // `overview` pane. let command = self.ignition.selected_command(); let selected = state.rack_state.selected; - info!(self.log, "Sending {command:?} to {selected}"); + info!( + self.log, + "Sending {command:?} to {}", + selected.to_string_uppercase() + ); self.popup = None; Some(Action::Ignition(selected, command)) } @@ -1378,7 +1400,10 @@ impl UpdatePane { // Draw the title/tab bar let title_bar = Paragraph::new(Line::from(vec![ Span::styled("UPDATE STATUS / ", border_style), - Span::styled(state.rack_state.selected.to_string(), header_style), + Span::styled( + state.rack_state.selected.to_string_uppercase(), + header_style, + ), ])) .block(block.clone()); frame.render_widget(title_bar, self.title_rect); @@ -1860,7 +1885,7 @@ impl ComponentUpdateListState { "root execution ID should have a summary associated with it", ); - match summary.execution_status { + match &summary.execution_status { ExecutionStatus::NotStarted => { status_text.push(Span::styled( "Update not started", @@ -1885,47 +1910,63 @@ impl ComponentUpdateListState { )); Some(ComponentUpdateShowHelp::Running) } - ExecutionStatus::Completed { .. } => { - status_text - .push(Span::styled("Update ", style::plain_text())); - status_text.push(Span::styled( - "completed", - style::successful_update_bold(), - )); - Some(ComponentUpdateShowHelp::Completed) - } - ExecutionStatus::Failed { step_key } => { - status_text - .push(Span::styled("Update ", style::plain_text())); - status_text.push(Span::styled( - "failed", - style::failed_update_bold(), - )); - status_text.push(Span::styled( - format!( - " at step {}/{}", - step_key.index + 1, - summary.total_steps, - ), - style::plain_text(), - )); - Some(ComponentUpdateShowHelp::Completed) - } - ExecutionStatus::Aborted { step_key } => { - status_text - .push(Span::styled("Update ", style::plain_text())); - status_text.push(Span::styled( - "aborted", - style::failed_update_bold(), - )); - status_text.push(Span::styled( - format!( - " at step {}/{}", - step_key.index + 1, - summary.total_steps, - ), - style::plain_text(), - )); + ExecutionStatus::Terminal(info) => { + match info.kind { + TerminalKind::Completed => { + status_text.push(Span::styled( + "Update ", + style::plain_text(), + )); + status_text.push(Span::styled( + "completed", + style::successful_update_bold(), + )); + } + TerminalKind::Failed => { + status_text.push(Span::styled( + "Update ", + style::plain_text(), + )); + status_text.push(Span::styled( + "failed", + style::failed_update_bold(), + )); + status_text.push(Span::styled( + format!( + " at step {}/{}", + info.step_key.index + 1, + summary.total_steps, + ), + style::plain_text(), + )); + } + TerminalKind::Aborted => { + status_text.push(Span::styled( + "Update ", + style::plain_text(), + )); + status_text.push(Span::styled( + "aborted", + style::failed_update_bold(), + )); + status_text.push(Span::styled( + format!( + " at step {}/{}", + info.step_key.index + 1, + summary.total_steps, + ), + style::plain_text(), + )); + } + } + + if let Some(total_elapsed) = info.root_total_elapsed { + status_text.push(Span::styled( + format!(" after {:.2?}", total_elapsed), + style::plain_text(), + )); + } + Some(ComponentUpdateShowHelp::Completed) } } diff --git a/wicket/src/ui/widgets/ignition.rs b/wicket/src/ui/widgets/ignition.rs index af0818e52a..cef942d2c7 100644 --- a/wicket/src/ui/widgets/ignition.rs +++ b/wicket/src/ui/widgets/ignition.rs @@ -58,7 +58,7 @@ impl IgnitionPopup { ) -> PopupBuilder<'static> { PopupBuilder { header: Line::from(vec![Span::styled( - format!("IGNITION: {}", component), + format!("IGNITION: {}", component.to_string_uppercase()), style::header(true), )]), body: Text { diff --git a/wicket/src/ui/wrap.rs b/wicket/src/ui/wrap.rs index 6cd5f7010a..9cd57d45d5 100644 --- a/wicket/src/ui/wrap.rs +++ b/wicket/src/ui/wrap.rs @@ -324,7 +324,7 @@ impl<'a> Fragment for StyledWord<'a> { /// Forcibly break spans wider than `line_width` into smaller spans. /// -/// This simply calls [`Span::break_apart`] on spans that are too wide. +/// This simply calls [`StyledWord::break_apart`] on spans that are too wide. fn break_words<'a, I>(spans: I, line_width: usize) -> Vec> where I: IntoIterator>, diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index 1044e1ff51..655f3bb803 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -7,6 +7,7 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true async-trait.workspace = true +base64.workspace = true bytes.workspace = true camino.workspace = true camino-tempfile.workspace = true @@ -24,6 +25,8 @@ hubtools.workspace = true http.workspace = true hyper.workspace = true illumos-utils.workspace = true +ipnetwork.workspace = true +internal-dns.workspace = true itertools.workspace = true reqwest.workspace = true schemars.workspace = true diff --git a/wicketd/src/artifacts/extracted_artifacts.rs b/wicketd/src/artifacts/extracted_artifacts.rs index f9ad59404b..352d8ad3d5 100644 --- a/wicketd/src/artifacts/extracted_artifacts.rs +++ b/wicketd/src/artifacts/extracted_artifacts.rs @@ -61,7 +61,7 @@ impl Eq for ExtractedArtifactDataHandle {} impl ExtractedArtifactDataHandle { /// File size of this artifact in bytes. - pub(super) fn file_size(&self) -> usize { + pub(crate) fn file_size(&self) -> usize { self.file_size } diff --git a/wicketd/src/artifacts/update_plan.rs b/wicketd/src/artifacts/update_plan.rs index 2668aaac51..31a8a06ca2 100644 --- a/wicketd/src/artifacts/update_plan.rs +++ b/wicketd/src/artifacts/update_plan.rs @@ -39,14 +39,14 @@ use tufaceous_lib::RotArchives; pub struct UpdatePlan { pub(crate) system_version: SemverVersion, pub(crate) gimlet_sp: BTreeMap, - pub(crate) gimlet_rot_a: ArtifactIdData, - pub(crate) gimlet_rot_b: ArtifactIdData, + pub(crate) gimlet_rot_a: Vec, + pub(crate) gimlet_rot_b: Vec, pub(crate) psc_sp: BTreeMap, - pub(crate) psc_rot_a: ArtifactIdData, - pub(crate) psc_rot_b: ArtifactIdData, + pub(crate) psc_rot_a: Vec, + pub(crate) psc_rot_b: Vec, pub(crate) sidecar_sp: BTreeMap, - pub(crate) sidecar_rot_a: ArtifactIdData, - pub(crate) sidecar_rot_b: ArtifactIdData, + pub(crate) sidecar_rot_a: Vec, + pub(crate) sidecar_rot_b: Vec, // Note: The Trampoline image is broken into phase1/phase2 as part of our // update plan (because they go to different destinations), but the two @@ -83,14 +83,14 @@ pub(super) struct UpdatePlanBuilder<'a> { // fields that mirror `UpdatePlan` system_version: SemverVersion, gimlet_sp: BTreeMap, - gimlet_rot_a: Option, - gimlet_rot_b: Option, + gimlet_rot_a: Vec, + gimlet_rot_b: Vec, psc_sp: BTreeMap, - psc_rot_a: Option, - psc_rot_b: Option, + psc_rot_a: Vec, + psc_rot_b: Vec, sidecar_sp: BTreeMap, - sidecar_rot_a: Option, - sidecar_rot_b: Option, + sidecar_rot_a: Vec, + sidecar_rot_b: Vec, // We always send phase 1 images (regardless of host or trampoline) to the // SP via MGS, so we retain their data. @@ -124,14 +124,14 @@ impl<'a> UpdatePlanBuilder<'a> { Ok(Self { system_version, gimlet_sp: BTreeMap::new(), - gimlet_rot_a: None, - gimlet_rot_b: None, + gimlet_rot_a: Vec::new(), + gimlet_rot_b: Vec::new(), psc_sp: BTreeMap::new(), - psc_rot_a: None, - psc_rot_b: None, + psc_rot_a: Vec::new(), + psc_rot_b: Vec::new(), sidecar_sp: BTreeMap::new(), - sidecar_rot_a: None, - sidecar_rot_b: None, + sidecar_rot_a: Vec::new(), + sidecar_rot_b: Vec::new(), host_phase_1: None, trampoline_phase_1: None, trampoline_phase_2: None, @@ -309,10 +309,6 @@ impl<'a> UpdatePlanBuilder<'a> { | KnownArtifactKind::SwitchSp => unreachable!(), }; - if rot_a.is_some() || rot_b.is_some() { - return Err(RepositoryError::DuplicateArtifactKind(artifact_kind)); - } - let (rot_a_data, rot_b_data) = Self::extract_nested_artifact_pair( &mut self.extracted_artifacts, artifact_kind, @@ -336,10 +332,8 @@ impl<'a> UpdatePlanBuilder<'a> { kind: rot_b_kind.clone(), }; - *rot_a = - Some(ArtifactIdData { id: rot_a_id, data: rot_a_data.clone() }); - *rot_b = - Some(ArtifactIdData { id: rot_b_id, data: rot_b_data.clone() }); + rot_a.push(ArtifactIdData { id: rot_a_id, data: rot_a_data.clone() }); + rot_b.push(ArtifactIdData { id: rot_b_id, data: rot_b_data.clone() }); record_extracted_artifact( artifact_id.clone(), @@ -574,53 +568,39 @@ impl<'a> UpdatePlanBuilder<'a> { pub(super) fn build(self) -> Result { // Ensure our multi-board-supporting kinds have at least one board // present. - if self.gimlet_sp.is_empty() { - return Err(RepositoryError::MissingArtifactKind( - KnownArtifactKind::GimletSp, - )); - } - if self.psc_sp.is_empty() { - return Err(RepositoryError::MissingArtifactKind( - KnownArtifactKind::PscSp, - )); - } - if self.sidecar_sp.is_empty() { - return Err(RepositoryError::MissingArtifactKind( - KnownArtifactKind::SwitchSp, - )); + for (kind, no_artifacts) in [ + (KnownArtifactKind::GimletSp, self.gimlet_sp.is_empty()), + (KnownArtifactKind::PscSp, self.psc_sp.is_empty()), + (KnownArtifactKind::SwitchSp, self.sidecar_sp.is_empty()), + ( + KnownArtifactKind::GimletRot, + self.gimlet_rot_a.is_empty() || self.gimlet_rot_b.is_empty(), + ), + ( + KnownArtifactKind::PscRot, + self.psc_rot_a.is_empty() || self.psc_rot_b.is_empty(), + ), + ( + KnownArtifactKind::SwitchRot, + self.sidecar_rot_a.is_empty() || self.sidecar_rot_b.is_empty(), + ), + ] { + if no_artifacts { + return Err(RepositoryError::MissingArtifactKind(kind)); + } } Ok(UpdatePlan { system_version: self.system_version, gimlet_sp: self.gimlet_sp, // checked above - gimlet_rot_a: self.gimlet_rot_a.ok_or( - RepositoryError::MissingArtifactKind( - KnownArtifactKind::GimletRot, - ), - )?, - gimlet_rot_b: self.gimlet_rot_b.ok_or( - RepositoryError::MissingArtifactKind( - KnownArtifactKind::GimletRot, - ), - )?, - psc_sp: self.psc_sp, // checked above - psc_rot_a: self.psc_rot_a.ok_or( - RepositoryError::MissingArtifactKind(KnownArtifactKind::PscRot), - )?, - psc_rot_b: self.psc_rot_b.ok_or( - RepositoryError::MissingArtifactKind(KnownArtifactKind::PscRot), - )?, + gimlet_rot_a: self.gimlet_rot_a, // checked above + gimlet_rot_b: self.gimlet_rot_b, // checked above + psc_sp: self.psc_sp, // checked above + psc_rot_a: self.psc_rot_a, // checked above + psc_rot_b: self.psc_rot_b, // checked above sidecar_sp: self.sidecar_sp, // checked above - sidecar_rot_a: self.sidecar_rot_a.ok_or( - RepositoryError::MissingArtifactKind( - KnownArtifactKind::SwitchRot, - ), - )?, - sidecar_rot_b: self.sidecar_rot_b.ok_or( - RepositoryError::MissingArtifactKind( - KnownArtifactKind::SwitchRot, - ), - )?, + sidecar_rot_a: self.sidecar_rot_a, // checked above + sidecar_rot_b: self.sidecar_rot_b, // checked above host_phase_1: self.host_phase_1.ok_or( RepositoryError::MissingArtifactKind(KnownArtifactKind::Host), )?, @@ -1030,21 +1010,27 @@ mod tests { // Check extracted RoT data assert_eq!( - read_to_vec(&plan.gimlet_rot_a.data).await, + read_to_vec(&plan.gimlet_rot_a[0].data).await, gimlet_rot.archive_a ); assert_eq!( - read_to_vec(&plan.gimlet_rot_b.data).await, + read_to_vec(&plan.gimlet_rot_b[0].data).await, gimlet_rot.archive_b ); - assert_eq!(read_to_vec(&plan.psc_rot_a.data).await, psc_rot.archive_a); - assert_eq!(read_to_vec(&plan.psc_rot_b.data).await, psc_rot.archive_b); assert_eq!( - read_to_vec(&plan.sidecar_rot_a.data).await, + read_to_vec(&plan.psc_rot_a[0].data).await, + psc_rot.archive_a + ); + assert_eq!( + read_to_vec(&plan.psc_rot_b[0].data).await, + psc_rot.archive_b + ); + assert_eq!( + read_to_vec(&plan.sidecar_rot_a[0].data).await, sidecar_rot.archive_a ); assert_eq!( - read_to_vec(&plan.sidecar_rot_b.data).await, + read_to_vec(&plan.sidecar_rot_b[0].data).await, sidecar_rot.archive_b ); diff --git a/wicketd/src/bin/wicketd.rs b/wicketd/src/bin/wicketd.rs index adfac5ac1a..2e6d51c0f0 100644 --- a/wicketd/src/bin/wicketd.rs +++ b/wicketd/src/bin/wicketd.rs @@ -5,11 +5,14 @@ //! Executable for wicketd: technician port based management service use clap::Parser; -use omicron_common::cmd::{fatal, CmdError}; +use omicron_common::{ + address::Ipv6Subnet, + cmd::{fatal, CmdError}, +}; use sled_hardware::Baseboard; -use std::net::SocketAddrV6; +use std::net::{Ipv6Addr, SocketAddrV6}; use std::path::PathBuf; -use wicketd::{self, run_openapi, Config, Server}; +use wicketd::{self, run_openapi, Config, Server, SmfConfigValues}; #[derive(Debug, Parser)] #[clap(name = "wicketd", about = "See README.adoc for more information")] @@ -30,12 +33,28 @@ enum Args { #[clap(long, action)] artifact_address: SocketAddrV6, - /// The port on localhost for MGS + /// The address (expected to be on localhost) for MGS #[clap(long, action)] mgs_address: SocketAddrV6, + /// The address (expected to be on localhost) on which we'll serve a TCP + /// proxy to Nexus's "techport external" API + #[clap(long, action)] + nexus_proxy_address: SocketAddrV6, + + /// Path to a file containing our baseboard information #[clap(long)] baseboard_file: Option, + + /// Read dynamic properties from our SMF config instead of passing them + /// on the command line + #[clap(long)] + read_smf_config: bool, + + /// The subnet for the rack; typically read directly from our SMF config + /// via `--read-smf-config` or an SMF refresh + #[clap(long, action, conflicts_with("read_smf_config"))] + rack_subnet: Option, }, } @@ -56,20 +75,22 @@ async fn do_run() -> Result<(), CmdError> { address, artifact_address, mgs_address, + nexus_proxy_address, baseboard_file, + read_smf_config, + rack_subnet, } => { let baseboard = if let Some(baseboard_file) = baseboard_file { - let baseboard_file = - std::fs::read_to_string(&baseboard_file) - .map_err(|e| CmdError::Failure(e.to_string()))?; + let baseboard_file = std::fs::read_to_string(baseboard_file) + .map_err(|e| CmdError::Failure(e.to_string()))?; let baseboard: Baseboard = serde_json::from_str(&baseboard_file) .map_err(|e| CmdError::Failure(e.to_string()))?; // TODO-correctness `Baseboard::unknown()` is slated for removal - // after some refactoring in sled-agent, at which point we'll need a - // different way for sled-agent to tell us it doesn't know our - // baseboard. + // after some refactoring in sled-agent, at which point we'll + // need a different way for sled-agent to tell us it doesn't + // know our baseboard. if matches!(baseboard, Baseboard::Unknown) { None } else { @@ -87,11 +108,23 @@ async fn do_run() -> Result<(), CmdError> { )) })?; + let rack_subnet = match rack_subnet { + Some(addr) => Some(Ipv6Subnet::new(addr)), + None if read_smf_config => { + let smf_values = SmfConfigValues::read_current() + .map_err(|e| CmdError::Failure(e.to_string()))?; + smf_values.rack_subnet + } + None => None, + }; + let args = wicketd::Args { address, artifact_address, mgs_address, + nexus_proxy_address, baseboard, + rack_subnet, }; let log = config.log.to_logger("wicketd").map_err(|msg| { CmdError::Failure(format!("initializing logger: {}", msg)) diff --git a/wicketd/src/context.rs b/wicketd/src/context.rs index db1c72fcb9..eeecc3fa64 100644 --- a/wicketd/src/context.rs +++ b/wicketd/src/context.rs @@ -13,6 +13,7 @@ use anyhow::anyhow; use anyhow::bail; use anyhow::Result; use gateway_client::types::SpIdentifier; +use internal_dns::resolver::Resolver; use sled_hardware::Baseboard; use slog::info; use std::net::Ipv6Addr; @@ -23,6 +24,7 @@ use std::sync::OnceLock; /// Shared state used by API handlers pub struct ServerContext { + pub(crate) bind_address: SocketAddrV6, pub mgs_handle: MgsHandle, pub mgs_client: gateway_client::Client, pub(crate) log: slog::Logger, @@ -36,6 +38,7 @@ pub struct ServerContext { pub(crate) baseboard: Option, pub(crate) rss_config: Mutex, pub(crate) preflight_checker: PreflightCheckerHandler, + pub(crate) internal_dns_resolver: Arc>>, } impl ServerContext { diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs index 72c3341334..be0f681601 100644 --- a/wicketd/src/http_entrypoints.rs +++ b/wicketd/src/http_entrypoints.rs @@ -12,6 +12,7 @@ use crate::mgs::MgsHandle; use crate::mgs::ShutdownInProgress; use crate::preflight_check::UplinkEventReport; use crate::RackV1Inventory; +use crate::SmfConfigValues; use bootstrap_agent_client::types::RackInitId; use bootstrap_agent_client::types::RackOperationStatus; use bootstrap_agent_client::types::RackResetId; @@ -29,6 +30,7 @@ use gateway_client::types::IgnitionCommand; use gateway_client::types::SpIdentifier; use gateway_client::types::SpType; use http::StatusCode; +use internal_dns::resolver::Resolver; use omicron_common::address; use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::shared::RackNetworkConfig; @@ -39,6 +41,7 @@ use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; use sled_hardware::Baseboard; +use slog::o; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::io; @@ -80,6 +83,7 @@ pub fn api() -> WicketdApiDescription { api.register(post_ignition_command)?; api.register(post_start_preflight_uplink_check)?; api.register(get_preflight_uplink_report)?; + api.register(post_reload_config)?; Ok(()) } @@ -1237,6 +1241,53 @@ async fn get_preflight_uplink_report( } } +/// An endpoint instructing wicketd to reload its SMF config properties. +/// +/// The only expected client of this endpoint is `curl` from wicketd's SMF +/// `refresh` method, but other clients hitting it is harmless. +#[endpoint { + method = POST, + path = "/reload-config", +}] +async fn post_reload_config( + rqctx: RequestContext, +) -> Result { + let smf_values = SmfConfigValues::read_current().map_err(|err| { + HttpError::for_unavail( + None, + format!("failed to read SMF values: {err}"), + ) + })?; + + let rqctx = rqctx.context(); + + // We do not allow a config reload to change our bound address; return an + // error if the caller is attempting to do so. + if rqctx.bind_address != smf_values.address { + return Err(HttpError::for_bad_request( + None, + "listening address cannot be reconfigured".to_string(), + )); + } + + if let Some(rack_subnet) = smf_values.rack_subnet { + let resolver = Resolver::new_from_subnet( + rqctx.log.new(o!("component" => "InternalDnsResolver")), + rack_subnet, + ) + .map_err(|err| { + HttpError::for_unavail( + None, + format!("failed to create internal DNS resolver: {err}"), + ) + })?; + + *rqctx.internal_dns_resolver.lock().unwrap() = Some(resolver); + } + + Ok(HttpResponseUpdatedNoContent()) +} + fn http_error_from_client_error( err: gateway_client::Error, ) -> HttpError { diff --git a/wicketd/src/installinator_progress.rs b/wicketd/src/installinator_progress.rs index ba3f743171..77baec2c94 100644 --- a/wicketd/src/installinator_progress.rs +++ b/wicketd/src/installinator_progress.rs @@ -165,7 +165,7 @@ enum RunningUpdate { /// Reports from the installinator have been received. /// /// This is an `UnboundedSender` to avoid cancel-safety issues (see - /// https://github.com/oxidecomputer/omicron/pull/3579). + /// ). ReportsReceived(watch::Sender), /// All messages have been received. diff --git a/wicketd/src/lib.rs b/wicketd/src/lib.rs index e17c15642c..ada1902654 100644 --- a/wicketd/src/lib.rs +++ b/wicketd/src/lib.rs @@ -11,6 +11,7 @@ mod http_entrypoints; mod installinator_progress; mod inventory; pub mod mgs; +mod nexus_proxy; mod preflight_check; mod rss_config; mod update_tracker; @@ -22,14 +23,17 @@ pub use config::Config; pub(crate) use context::ServerContext; use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer}; pub use installinator_progress::{IprUpdateTracker, RunningUpdateState}; +use internal_dns::resolver::Resolver; pub use inventory::{RackV1Inventory, SpInventory}; use mgs::make_mgs_client; pub(crate) use mgs::{MgsHandle, MgsManager}; +use nexus_proxy::NexusTcpProxy; +use omicron_common::address::{Ipv6Subnet, AZ_PREFIX}; use omicron_common::FileKv; use preflight_check::PreflightCheckerHandler; use sled_hardware::Baseboard; use slog::{debug, error, o, Drain}; -use std::sync::OnceLock; +use std::sync::{Mutex, OnceLock}; use std::{ net::{SocketAddr, SocketAddrV6}, sync::Arc, @@ -53,7 +57,62 @@ pub struct Args { pub address: SocketAddrV6, pub artifact_address: SocketAddrV6, pub mgs_address: SocketAddrV6, + pub nexus_proxy_address: SocketAddrV6, pub baseboard: Option, + pub rack_subnet: Option>, +} + +pub struct SmfConfigValues { + pub address: SocketAddrV6, + pub rack_subnet: Option>, +} + +impl SmfConfigValues { + #[cfg(target_os = "illumos")] + pub fn read_current() -> Result { + use anyhow::Context; + use illumos_utils::scf::ScfHandle; + + const CONFIG_PG: &str = "config"; + const PROP_RACK_SUBNET: &str = "rack-subnet"; + const PROP_ADDRESS: &str = "address"; + + let scf = ScfHandle::new()?; + let instance = scf.self_instance()?; + let snapshot = instance.running_snapshot()?; + let config = snapshot.property_group(CONFIG_PG)?; + + let rack_subnet = config.value_as_string(PROP_RACK_SUBNET)?; + + let rack_subnet = if rack_subnet == "unknown" { + None + } else { + let addr = rack_subnet.parse().with_context(|| { + format!( + "failed to parse {CONFIG_PG}/{PROP_RACK_SUBNET} \ + value {rack_subnet:?} as an IP address" + ) + })?; + Some(Ipv6Subnet::new(addr)) + }; + + let address = { + let address = config.value_as_string(PROP_ADDRESS)?; + address.parse().with_context(|| { + format!( + "failed to parse {CONFIG_PG}/{PROP_ADDRESS} \ + value {address:?} as a socket address" + ) + })? + }; + + Ok(Self { address, rack_subnet }) + } + + #[cfg(not(target_os = "illumos"))] + pub fn read_current() -> Result { + Err(anyhow!("reading SMF config only available on illumos")) + } } pub struct Server { @@ -62,6 +121,7 @@ pub struct Server { pub artifact_store: WicketdArtifactStore, pub update_tracker: Arc, pub ipr_update_tracker: IprUpdateTracker, + nexus_tcp_proxy: NexusTcpProxy, } impl Server { @@ -80,8 +140,9 @@ impl Server { let dropshot_config = ConfigDropshot { bind_address: SocketAddr::V6(args.address), - // The maximum request size is set to 4 GB -- artifacts can be large and there's currently - // no way to set a larger request size for some endpoints. + // The maximum request size is set to 4 GB -- artifacts can be large + // and there's currently no way to set a larger request size for + // some endpoints. request_body_max_bytes: 4 << 30, default_handler_task_mode: HandlerTaskMode::Detached, }; @@ -104,6 +165,27 @@ impl Server { )); let bootstrap_peers = BootstrapPeers::new(&log); + let internal_dns_resolver = args + .rack_subnet + .map(|addr| { + Resolver::new_from_subnet( + log.new(o!("component" => "InternalDnsResolver")), + addr, + ) + .map_err(|err| { + format!("Could not create internal DNS resolver: {err}") + }) + }) + .transpose()?; + + let internal_dns_resolver = Arc::new(Mutex::new(internal_dns_resolver)); + let nexus_tcp_proxy = NexusTcpProxy::start( + args.nexus_proxy_address, + Arc::clone(&internal_dns_resolver), + &log, + ) + .await + .map_err(|err| format!("failed to start Nexus TCP proxy: {err}"))?; let wicketd_server = { let ds_log = log.new(o!("component" => "dropshot (wicketd)")); @@ -112,6 +194,7 @@ impl Server { &dropshot_config, http_entrypoints::api(), ServerContext { + bind_address: args.address, mgs_handle, mgs_client, log: log.clone(), @@ -121,6 +204,7 @@ impl Server { baseboard: args.baseboard, rss_config: Default::default(), preflight_checker: PreflightCheckerHandler::new(&log), + internal_dns_resolver, }, &ds_log, ) @@ -146,17 +230,19 @@ impl Server { artifact_store: store, update_tracker, ipr_update_tracker, + nexus_tcp_proxy, }) } /// Close all running dropshot servers. - pub async fn close(self) -> Result<()> { + pub async fn close(mut self) -> Result<()> { self.wicketd_server.close().await.map_err(|error| { anyhow!("error closing wicketd server: {error}") })?; self.artifact_server.close().await.map_err(|error| { anyhow!("error closing artifact server: {error}") })?; + self.nexus_tcp_proxy.shutdown(); Ok(()) } diff --git a/wicketd/src/nexus_proxy.rs b/wicketd/src/nexus_proxy.rs new file mode 100644 index 0000000000..33ff02a945 --- /dev/null +++ b/wicketd/src/nexus_proxy.rs @@ -0,0 +1,178 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! TCP proxy to expose Nexus's external API via the techport. + +use internal_dns::resolver::Resolver; +use internal_dns::ServiceName; +use omicron_common::address::NEXUS_TECHPORT_EXTERNAL_PORT; +use slog::info; +use slog::o; +use slog::warn; +use slog::Logger; +use std::io; +use std::net::SocketAddr; +use std::net::SocketAddrV6; +use std::sync::Arc; +use std::sync::Mutex; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio::sync::oneshot; + +pub(crate) struct NexusTcpProxy { + shutdown_tx: Option>, +} + +impl Drop for NexusTcpProxy { + fn drop(&mut self) { + self.shutdown(); + } +} + +impl NexusTcpProxy { + pub(crate) async fn start( + listen_addr: SocketAddrV6, + internal_dns_resolver: Arc>>, + log: &Logger, + ) -> io::Result { + let listener = TcpListener::bind(listen_addr).await?; + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + + let inner = Inner { + listener, + internal_dns_resolver, + shutdown_rx, + log: log.new(o!("component" => "NexusTcpProxy")), + }; + + tokio::spawn(inner.run()); + + Ok(Self { shutdown_tx: Some(shutdown_tx) }) + } + + pub(crate) fn shutdown(&mut self) { + if let Some(tx) = self.shutdown_tx.take() { + // We want to shutdown the task we spawned; failure to message it + // means it's already done. + _ = tx.send(()); + } + } +} + +struct Inner { + listener: TcpListener, + internal_dns_resolver: Arc>>, + shutdown_rx: oneshot::Receiver<()>, + log: Logger, +} + +impl Inner { + async fn run(mut self) { + loop { + tokio::select! { + // Cancel-safe per the docs on `TcpListener::accept()` + result = self.listener.accept() => { + self.spawn_proxy_handler(result); + } + + // Cancel-safe: awaiting a `&mut Fut` does not drop the future + _ = &mut self.shutdown_rx => { + info!(self.log, "exiting"); + return; + } + } + } + } + + fn spawn_proxy_handler( + &mut self, + result: io::Result<(TcpStream, SocketAddr)>, + ) { + let (stream, log) = match result { + Ok((stream, peer)) => (stream, self.log.new(o!("peer" => peer))), + Err(err) => { + warn!(self.log, "accept() failed"; "err" => %err); + return; + } + }; + + info!(log, "accepted connection"); + + let Some(resolver) = self.internal_dns_resolver.lock().unwrap().clone() + else { + info!( + log, + "closing connection; no internal DNS resolver available \ + (rack subnet unknown?)" + ); + return; + }; + + tokio::spawn(run_proxy(stream, resolver, log)); + } +} + +async fn run_proxy( + mut client_stream: TcpStream, + resolver: Resolver, + log: Logger, +) { + // Can we talk to the internal DNS server(s) to find Nexus's IPs? + let nexus_addrs = match resolver.lookup_all_ipv6(ServiceName::Nexus).await { + Ok(ips) => ips + .into_iter() + .map(|ip| { + SocketAddr::V6(SocketAddrV6::new( + ip, + NEXUS_TECHPORT_EXTERNAL_PORT, + 0, + 0, + )) + }) + .collect::>(), + Err(err) => { + warn!( + log, "failed to look up Nexus IP addrs"; + "err" => %err, + ); + return; + } + }; + + // Can we connect to any Nexus instance? + let mut nexus_stream = + match TcpStream::connect(nexus_addrs.as_slice()).await { + Ok(stream) => stream, + Err(err) => { + warn!( + log, "failed to connect to Nexus"; + "nexus_addrs" => ?nexus_addrs, + "err" => %err, + ); + return; + } + }; + + let log = match nexus_stream.peer_addr() { + Ok(addr) => log.new(o!("nexus_addr" => addr)), + Err(err) => log.new(o!("nexus_addr" => + format!("failed to read Nexus peer addr: {err}"))), + }; + info!(log, "connected to Nexus"); + + match tokio::io::copy_bidirectional(&mut client_stream, &mut nexus_stream) + .await + { + Ok((client_to_nexus, nexus_to_client)) => { + info!( + log, "closing successful proxy connection to Nexus"; + "bytes_sent_to_nexus" => client_to_nexus, + "bytes_sent_to_client" => nexus_to_client, + ); + } + Err(err) => { + warn!(log, "error proxying data to Nexus"; "err" => %err); + } + } +} diff --git a/wicketd/src/preflight_check/uplink.rs b/wicketd/src/preflight_check/uplink.rs index 58955d04d6..4d199d28b8 100644 --- a/wicketd/src/preflight_check/uplink.rs +++ b/wicketd/src/preflight_check/uplink.rs @@ -17,12 +17,13 @@ use dpd_client::ClientState as DpdClientState; use either::Either; use illumos_utils::zone::SVCCFG; use illumos_utils::PFEXEC; +use ipnetwork::IpNetwork; use omicron_common::address::DENDRITE_PORT; +use omicron_common::api::internal::shared::PortConfigV1; use omicron_common::api::internal::shared::PortFec as OmicronPortFec; use omicron_common::api::internal::shared::PortSpeed as OmicronPortSpeed; use omicron_common::api::internal::shared::RackNetworkConfig; use omicron_common::api::internal::shared::SwitchLocation; -use omicron_common::api::internal::shared::UplinkConfig; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -32,7 +33,6 @@ use slog::Logger; use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; -use std::net::Ipv4Addr; use std::str::FromStr; use std::sync::Arc; use std::sync::Mutex; @@ -66,8 +66,6 @@ const CHRONYD: &str = "/usr/sbin/chronyd"; const IPADM: &str = "/usr/sbin/ipadm"; const ROUTE: &str = "/usr/sbin/route"; -const DPD_DEFAULT_IPV4_CIDR: &str = "0.0.0.0/0"; - pub(super) async fn run_local_uplink_preflight_check( network_config: RackNetworkConfig, dns_servers: Vec, @@ -90,7 +88,7 @@ pub(super) async fn run_local_uplink_preflight_check( let mut engine = UpdateEngine::new(log, sender); for uplink in network_config - .uplinks + .ports .iter() .filter(|uplink| uplink.switch == our_switch_location) { @@ -131,7 +129,7 @@ pub(super) async fn run_local_uplink_preflight_check( fn add_steps_for_single_local_uplink_preflight_check<'a>( engine: &mut UpdateEngine<'a>, dpd_client: &'a DpdClient, - uplink: &'a UplinkConfig, + uplink: &'a PortConfigV1, dns_servers: &'a [IpAddr], ntp_servers: &'a [String], dns_name_to_query: Option<&'a str>, @@ -153,7 +151,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( // Timeout we give to chronyd during the NTP check, in seconds. const CHRONYD_CHECK_TIMEOUT_SECS: &str = "30"; - let registrar = engine.for_component(uplink.uplink_port.clone()); + let registrar = engine.for_component(uplink.port.clone()); let prev_step = registrar .new_step( @@ -162,7 +160,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( |_cx| async { // Check that the port name is valid and that it has no links // configured already. - let port_id = PortId::from_str(&uplink.uplink_port) + let port_id = PortId::from_str(&uplink.port) .map_err(UplinkPreflightTerminalError::InvalidPortName)?; let links = dpd_client .link_list(&port_id) @@ -192,11 +190,8 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( { Ok(_response) => { let metadata = vec![format!( - "configured {}/{}: ip {}, gateway {}", - *port_id, - link_id.0, - uplink.uplink_cidr, - uplink.gateway_ip + "configured {:?}/{}: ips {:#?}, routes {:#?}", + port_id, link_id.0, uplink.addresses, uplink.routes )]; StepSuccess::new((port_id, link_id)) .with_metadata(metadata) @@ -298,93 +293,99 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( // Tell the `uplink` service about the IP address we created on // the switch when configuring the uplink. let uplink_property = - UplinkProperty(format!("uplinks/{}_0", uplink.uplink_port)); - let uplink_cidr = uplink.uplink_cidr.to_string(); - - if let Err(err) = execute_command(&[ - SVCCFG, - "-s", - UPLINK_SMF_NAME, - "addpropvalue", - &uplink_property.0, - "astring:", - &uplink_cidr, - ]) - .await - { - return StepWarning::new( - Err(L2Failure::UplinkAddProperty(level1)), - format!("could not add uplink property: {err}"), - ) - .into(); - }; - - if let Err(err) = execute_command(&[ - SVCCFG, - "-s", - UPLINK_DEFAULT_SMF_NAME, - "refresh", - ]) - .await - { - return StepWarning::new( - Err(L2Failure::UplinkRefresh(level1, uplink_property)), - format!("could not add uplink property: {err}"), - ) - .into(); - }; - - // Wait for the `uplink` service to create the IP address. - let start_waiting_addr = Instant::now(); - 'waiting_for_addr: loop { - let ipadm_out = match execute_command(&[ - IPADM, - "show-addr", - "-p", - "-o", - "addr", + UplinkProperty(format!("uplinks/{}_0", uplink.port)); + + for addr in &uplink.addresses { + let uplink_cidr = addr.to_string(); + if let Err(err) = execute_command(&[ + SVCCFG, + "-s", + UPLINK_SMF_NAME, + "addpropvalue", + &uplink_property.0, + "astring:", + &uplink_cidr, ]) .await { - Ok(stdout) => stdout, - Err(err) => { - return StepWarning::new( - Err(L2Failure::RunIpadm( - level1, - uplink_property, - )), - format!("failed running ipadm: {err}"), - ) - .into(); - } + return StepWarning::new( + Err(L2Failure::UplinkAddProperty(level1)), + format!("could not add uplink property: {err}"), + ) + .into(); }; - for line in ipadm_out.split('\n') { - if line == uplink_cidr { - break 'waiting_for_addr; - } - } - - // We did not find `uplink_cidr` in the output of ipadm; - // sleep a bit and try again, unless we've been waiting too - // long already. - if start_waiting_addr.elapsed() < UPLINK_SVC_WAIT_TIMEOUT { - tokio::time::sleep(UPLINK_SVC_RETRY_DELAY).await; - } else { + if let Err(err) = execute_command(&[ + SVCCFG, + "-s", + UPLINK_DEFAULT_SMF_NAME, + "refresh", + ]) + .await + { return StepWarning::new( - Err(L2Failure::WaitingForHostAddr( + Err(L2Failure::UplinkRefresh( level1, uplink_property, )), - format!( - "timed out waiting for `uplink` to \ - create {uplink_cidr}" - ), + format!("could not add uplink property: {err}"), ) .into(); + }; + + // Wait for the `uplink` service to create the IP address. + let start_waiting_addr = Instant::now(); + 'waiting_for_addr: loop { + let ipadm_out = match execute_command(&[ + IPADM, + "show-addr", + "-p", + "-o", + "addr", + ]) + .await + { + Ok(stdout) => stdout, + Err(err) => { + return StepWarning::new( + Err(L2Failure::RunIpadm( + level1, + uplink_property, + )), + format!("failed running ipadm: {err}"), + ) + .into(); + } + }; + + for line in ipadm_out.split('\n') { + if line == uplink_cidr { + break 'waiting_for_addr; + } + } + + // We did not find `uplink_cidr` in the output of ipadm; + // sleep a bit and try again, unless we've been waiting too + // long already. + if start_waiting_addr.elapsed() + < UPLINK_SVC_WAIT_TIMEOUT + { + tokio::time::sleep(UPLINK_SVC_RETRY_DELAY).await; + } else { + return StepWarning::new( + Err(L2Failure::WaitingForHostAddr( + level1, + uplink_property, + )), + format!( + "timed out waiting for `uplink` to \ + create {uplink_cidr}" + ), + ) + .into(); + } } } - let metadata = vec![format!("configured {}", uplink_property.0)]; StepSuccess::new(Ok(L2Success { level1, uplink_property })) @@ -410,27 +411,29 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } }; - // Add the gateway as the default route in illumos. - if let Err(err) = execute_command(&[ - ROUTE, - "add", - "-inet", - "default", - &uplink.gateway_ip.to_string(), - ]) - .await - { - return StepWarning::new( - Err(RoutingFailure::HostDefaultRoute(level2)), - format!("could not add default route: {err}"), - ) - .into(); - }; + for r in &uplink.routes { + // Add the gateway as the default route in illumos. + if let Err(err) = execute_command(&[ + ROUTE, + "add", + "-inet", + &r.destination.to_string(), + &r.nexthop.to_string(), + ]) + .await + { + return StepWarning::new( + Err(RoutingFailure::HostDefaultRoute(level2)), + format!("could not add default route: {err}"), + ) + .into(); + }; + } StepSuccess::new(Ok(RoutingSuccess { level2 })) .with_metadata(vec![format!( - "added default route to {}", - uplink.gateway_ip + "added routes {:#?}", + uplink.routes, )]) .into() }, @@ -595,21 +598,24 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } }; - if remove_host_route { - execute_command(&[ - ROUTE, - "delete", - "-inet", - "default", - &uplink.gateway_ip.to_string(), - ]) - .await - .map_err(|err| { - UplinkPreflightTerminalError::RemoveHostRoute { - err, - gateway_ip: uplink.gateway_ip, - } - })?; + for r in &uplink.routes { + if remove_host_route { + execute_command(&[ + ROUTE, + "delete", + "-inet", + &r.destination.to_string(), + &r.nexthop.to_string(), + ]) + .await + .map_err(|err| { + UplinkPreflightTerminalError::RemoveHostRoute { + err, + destination: r.destination, + nexthop: r.nexthop, + } + })?; + } } StepSuccess::new(Ok(level2)).into() @@ -730,7 +736,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } fn build_port_settings( - uplink: &UplinkConfig, + uplink: &PortConfigV1, link_id: &LinkId, ) -> PortSettings { // Map from omicron_common types to dpd_client types @@ -758,10 +764,12 @@ fn build_port_settings( v6_routes: HashMap::new(), }; + let addrs = uplink.addresses.iter().map(|a| a.ip()).collect(); + port_settings.links.insert( link_id.to_string(), LinkSettings { - addrs: vec![IpAddr::V4(uplink.uplink_cidr.ip())], + addrs, params: LinkCreate { // TODO we should take these parameters too // https://github.com/oxidecomputer/omicron/issues/3061 @@ -773,14 +781,16 @@ fn build_port_settings( }, ); - port_settings.v4_routes.insert( - DPD_DEFAULT_IPV4_CIDR.parse().unwrap(), - RouteSettingsV4 { - link_id: link_id.0, - nexthop: uplink.gateway_ip, - vid: uplink.uplink_vid, - }, - ); + for r in &uplink.routes { + if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) = + (r.destination, r.nexthop) + { + port_settings.v4_routes.insert( + dst.to_string(), + vec![RouteSettingsV4 { link_id: link_id.0, nexthop }], + ); + } + } port_settings } @@ -890,8 +900,10 @@ pub(crate) enum UplinkPreflightTerminalError { err: DpdError, port_id: PortId, }, - #[error("failed to remove host OS route to gateway {gateway_ip}: {err}")] - RemoveHostRoute { err: String, gateway_ip: Ipv4Addr }, + #[error( + "failed to remove host OS route {destination} -> {nexthop}: {err}" + )] + RemoveHostRoute { err: String, destination: IpNetwork, nexthop: IpAddr }, #[error("failed to remove uplink SMF property {property:?}: {err}")] RemoveSmfProperty { property: String, err: String }, #[error("failed to refresh uplink service config: {0}")] diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index 1dc9f84985..a96acc56a0 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -453,18 +453,21 @@ impl From<&'_ CurrentRssConfig> for CurrentRssUserConfig { fn validate_rack_network_config( config: &RackNetworkConfig, -) -> Result { +) -> Result { + use bootstrap_agent_client::types::BgpConfig as BaBgpConfig; + use bootstrap_agent_client::types::BgpPeerConfig as BaBgpPeerConfig; + use bootstrap_agent_client::types::PortConfigV1 as BaPortConfigV1; use bootstrap_agent_client::types::PortFec as BaPortFec; use bootstrap_agent_client::types::PortSpeed as BaPortSpeed; + use bootstrap_agent_client::types::RouteConfig as BaRouteConfig; use bootstrap_agent_client::types::SwitchLocation as BaSwitchLocation; - use bootstrap_agent_client::types::UplinkConfig as BaUplinkConfig; use omicron_common::api::internal::shared::PortFec; use omicron_common::api::internal::shared::PortSpeed; use omicron_common::api::internal::shared::SwitchLocation; // Ensure that there is at least one uplink - if config.uplinks.is_empty() { - return Err(anyhow!("Must have at least one uplink configured")); + if config.ports.is_empty() { + return Err(anyhow!("Must have at least one port configured")); } // Make sure `infra_ip_first`..`infra_ip_last` is a well-defined range... @@ -475,34 +478,55 @@ fn validate_rack_network_config( }, )?; - // iterate through each UplinkConfig - for uplink_config in &config.uplinks { - // ... and check that it contains `uplink_ip`. - if uplink_config.uplink_cidr.ip() < infra_ip_range.first - || uplink_config.uplink_cidr.ip() > infra_ip_range.last - { - bail!( + // TODO this implies a single contiguous range for port IPs which is over + // constraining + // iterate through each port config + for port_config in &config.ports { + for addr in &port_config.addresses { + // ... and check that it contains `uplink_ip`. + if addr.ip() < infra_ip_range.first + || addr.ip() > infra_ip_range.last + { + bail!( "`uplink_cidr`'s IP address must be in the range defined by \ `infra_ip_first` and `infra_ip_last`" ); + } } } // TODO Add more client side checks on `rack_network_config` contents? - Ok(bootstrap_agent_client::types::RackNetworkConfig { + Ok(bootstrap_agent_client::types::RackNetworkConfigV1 { + rack_subnet: config.rack_subnet, infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, - uplinks: config - .uplinks + ports: config + .ports .iter() - .map(|config| BaUplinkConfig { - gateway_ip: config.gateway_ip, + .map(|config| BaPortConfigV1 { + port: config.port.clone(), + routes: config + .routes + .iter() + .map(|r| BaRouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), + bgp_peers: config + .bgp_peers + .iter() + .map(|p| BaBgpPeerConfig { + addr: p.addr, + asn: p.asn, + port: p.port.clone(), + }) + .collect(), switch: match config.switch { SwitchLocation::Switch0 => BaSwitchLocation::Switch0, SwitchLocation::Switch1 => BaSwitchLocation::Switch1, }, - uplink_cidr: config.uplink_cidr, - uplink_port: config.uplink_port.clone(), uplink_port_speed: match config.uplink_port_speed { PortSpeed::Speed0G => BaPortSpeed::Speed0G, PortSpeed::Speed1G => BaPortSpeed::Speed1G, @@ -519,7 +543,14 @@ fn validate_rack_network_config( PortFec::None => BaPortFec::None, PortFec::Rs => BaPortFec::Rs, }, - uplink_vid: config.uplink_vid, + }) + .collect(), + bgp: config + .bgp + .iter() + .map(|config| BaBgpConfig { + asn: config.asn, + originate: config.originate.clone(), }) .collect(), }) diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 05f57935a2..bd8e187fe9 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -18,18 +18,23 @@ use anyhow::anyhow; use anyhow::bail; use anyhow::ensure; use anyhow::Context; +use base64::Engine; use display_error_chain::DisplayErrorChain; use dropshot::HttpError; +use futures::TryFutureExt; use gateway_client::types::HostPhase2Progress; use gateway_client::types::HostPhase2RecoveryImageId; use gateway_client::types::HostStartupOptions; use gateway_client::types::InstallinatorImageId; use gateway_client::types::PowerState; +use gateway_client::types::RotCfpaSlot; use gateway_client::types::SpComponentFirmwareSlot; use gateway_client::types::SpIdentifier; use gateway_client::types::SpType; use gateway_client::types::SpUpdateStatus; use gateway_messages::SpComponent; +use gateway_messages::ROT_PAGE_SIZE; +use hubtools::RawHubrisArchive; use installinator_common::InstallinatorCompletionMetadata; use installinator_common::InstallinatorSpec; use installinator_common::M2Slot; @@ -52,11 +57,13 @@ use std::sync::Mutex as StdMutex; use std::time::Duration; use std::time::Instant; use thiserror::Error; +use tokio::io::AsyncReadExt; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::sync::watch; use tokio::sync::Mutex; use tokio::task::JoinHandle; +use tokio_util::io::StreamReader; use update_engine::events::ProgressUnits; use update_engine::AbortHandle; use update_engine::StepSpec; @@ -768,19 +775,13 @@ impl UpdateDriver { } let (rot_a, rot_b, sp_artifacts) = match update_cx.sp.type_ { - SpType::Sled => ( - plan.gimlet_rot_a.clone(), - plan.gimlet_rot_b.clone(), - &plan.gimlet_sp, - ), - SpType::Power => { - (plan.psc_rot_a.clone(), plan.psc_rot_b.clone(), &plan.psc_sp) + SpType::Sled => { + (&plan.gimlet_rot_a, &plan.gimlet_rot_b, &plan.gimlet_sp) + } + SpType::Power => (&plan.psc_rot_a, &plan.psc_rot_b, &plan.psc_sp), + SpType::Switch => { + (&plan.sidecar_rot_a, &plan.sidecar_rot_b, &plan.sidecar_sp) } - SpType::Switch => ( - plan.sidecar_rot_a.clone(), - plan.sidecar_rot_b.clone(), - &plan.sidecar_sp, - ), }; let rot_registrar = engine.for_component(UpdateComponent::Rot); @@ -790,16 +791,15 @@ impl UpdateDriver { // currently executing; we must update the _other_ slot. We also want to // know its current version (so we can skip updating if we only need to // update the SP and/or host). - let rot_interrogation = - rot_registrar - .new_step( - UpdateStepId::InterrogateRot, - "Checking current RoT version and active slot", - |_cx| async move { - update_cx.interrogate_rot(rot_a, rot_b).await - }, - ) - .register(); + let rot_interrogation = rot_registrar + .new_step( + UpdateStepId::InterrogateRot, + "Checking current RoT version and active slot", + move |_cx| async move { + update_cx.interrogate_rot(rot_a, rot_b).await + }, + ) + .register(); // The SP only has one updateable firmware slot ("the inactive bank"). // We want to ask about slot 0 (the active slot)'s current version, and @@ -1553,8 +1553,8 @@ impl UpdateContext { async fn interrogate_rot( &self, - rot_a: ArtifactIdData, - rot_b: ArtifactIdData, + rot_a: &[ArtifactIdData], + rot_b: &[ArtifactIdData], ) -> Result, UpdateTerminalError> { let rot_active_slot = self .get_component_active_slot(SpComponent::ROT.const_as_str()) @@ -1565,7 +1565,7 @@ impl UpdateContext { // Flip these around: if 0 (A) is active, we want to // update 1 (B), and vice versa. - let (active_slot_name, slot_to_update, artifact_to_apply) = + let (active_slot_name, slot_to_update, available_artifacts) = match rot_active_slot { 0 => ('A', 1, rot_b), 1 => ('B', 0, rot_a), @@ -1578,6 +1578,176 @@ impl UpdateContext { } }; + // Read the CMPA and currently-active CFPA so we can find the + // correctly-signed artifact. + let base64_decode_rot_page = |data: String| { + // Even though we know `data` should decode to exactly + // `ROT_PAGE_SIZE` bytes, the base64 crate requires an output buffer + // of at least `decoded_len_estimate`. Allocate such a buffer here, + // then we'll copy to the fixed-size array we need after confirming + // the number of decoded bytes; + let mut output_buf = + vec![0; base64::decoded_len_estimate(data.len())]; + + let n = base64::engine::general_purpose::STANDARD + .decode_slice(&data, &mut output_buf) + .with_context(|| { + format!("failed to decode base64 string: {data:?}") + })?; + if n != ROT_PAGE_SIZE { + bail!( + "incorrect len ({n}, expected {ROT_PAGE_SIZE}) \ + after decoding base64 string: {data:?}", + ); + } + let mut page = [0; ROT_PAGE_SIZE]; + page.copy_from_slice(&output_buf[..n]); + Ok(page) + }; + + // We may be talking to an SP / RoT that doesn't yet know how to give us + // its CMPA. If we hit a protocol version error here, we can fall back + // to our old behavior of requiring exactly 1 RoT artifact. + let mut artifact_to_apply = None; + + let cmpa = match self + .mgs_client + .sp_rot_cmpa_get( + self.sp.type_, + self.sp.slot, + SpComponent::ROT.const_as_str(), + ) + .await + { + Ok(response) => { + let data = response.into_inner().base64_data; + Some(base64_decode_rot_page(data).map_err(|error| { + UpdateTerminalError::GetRotCmpaFailed { error } + })?) + } + // TODO is there a better way to check the _specific_ error response + // we get here? We only have a couple of strings; we could check the + // error string contents for something like "WrongVersion", but + // that's pretty fragile. Instead we'll treat any error response + // here as a "fallback to previous behavior". + Err(err @ gateway_client::Error::ErrorResponse(_)) => { + if available_artifacts.len() == 1 { + info!( + self.log, + "Failed to get RoT CMPA page; \ + using only available RoT artifact"; + "err" => %err, + ); + artifact_to_apply = Some(available_artifacts[0].clone()); + None + } else { + error!( + self.log, + "Failed to get RoT CMPA; unable to choose from \ + multiple available RoT artifacts"; + "err" => %err, + "num_rot_artifacts" => available_artifacts.len(), + ); + return Err(UpdateTerminalError::GetRotCmpaFailed { + error: err.into(), + }); + } + } + // For any other error (e.g., comms failures), just fail as normal. + Err(err) => { + return Err(UpdateTerminalError::GetRotCmpaFailed { + error: err.into(), + }); + } + }; + + // If we were able to fetch the CMPA, we also need to fetch the CFPA and + // then find the correct RoT artifact. If we weren't able to fetch the + // CMPA, we either already set `artifact_to_apply` to the one-and-only + // RoT artifact available, or we returned a terminal error. + if let Some(cmpa) = cmpa { + let cfpa = self + .mgs_client + .sp_rot_cfpa_get( + self.sp.type_, + self.sp.slot, + SpComponent::ROT.const_as_str(), + &gateway_client::types::GetCfpaParams { + slot: RotCfpaSlot::Active, + }, + ) + .await + .map_err(|err| UpdateTerminalError::GetRotCfpaFailed { + error: err.into(), + }) + .and_then(|response| { + let data = response.into_inner().base64_data; + base64_decode_rot_page(data).map_err(|error| { + UpdateTerminalError::GetRotCfpaFailed { error } + }) + })?; + + // Loop through our possible artifacts and find the first (we only + // expect one!) that verifies against the RoT's CMPA/CFPA. + for artifact in available_artifacts { + let image = artifact + .data + .reader_stream() + .and_then(|stream| async { + let mut buf = + Vec::with_capacity(artifact.data.file_size()); + StreamReader::new(stream) + .read_to_end(&mut buf) + .await + .context("I/O error reading extracted archive")?; + Ok(buf) + }) + .await + .map_err(|error| { + UpdateTerminalError::FailedFindingSignedRotImage { + error, + } + })?; + let archive = + RawHubrisArchive::from_vec(image).map_err(|err| { + UpdateTerminalError::FailedFindingSignedRotImage { + error: anyhow::Error::new(err).context(format!( + "failed to read hubris archive for {:?}", + artifact.id + )), + } + })?; + match archive.verify(&cmpa, &cfpa) { + Ok(()) => { + info!( + self.log, "RoT archive verification success"; + "name" => artifact.id.name.as_str(), + "version" => %artifact.id.version, + "kind" => ?artifact.id.kind, + ); + artifact_to_apply = Some(artifact.clone()); + break; + } + Err(err) => { + // We log this but don't fail - we want to continue + // looking for a verifiable artifact. + info!( + self.log, "RoT archive verification failed"; + "artifact" => ?artifact.id, + "err" => %DisplayErrorChain::new(&err), + ); + } + } + } + } + + // Ensure we found a valid RoT artifact. + let artifact_to_apply = artifact_to_apply.ok_or_else(|| { + UpdateTerminalError::FailedFindingSignedRotImage { + error: anyhow!("no RoT image found with valid CMPA/CFPA"), + } + })?; + // Read the caboose of the currently-active slot. let caboose = self .mgs_client diff --git a/wicketd/tests/integration_tests/setup.rs b/wicketd/tests/integration_tests/setup.rs index f0ca183c6a..62682a73ab 100644 --- a/wicketd/tests/integration_tests/setup.rs +++ b/wicketd/tests/integration_tests/setup.rs @@ -40,7 +40,9 @@ impl WicketdTestContext { address: localhost_port_0, artifact_address: localhost_port_0, mgs_address, + nexus_proxy_address: localhost_port_0, baseboard: None, + rack_subnet: None, }; let server = wicketd::Server::start(log.clone(), args) diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs index a198068ef3..a9be9d4747 100644 --- a/wicketd/tests/integration_tests/updates.rs +++ b/wicketd/tests/integration_tests/updates.rs @@ -168,8 +168,7 @@ async fn test_updates() { match terminal_event.kind { StepEventKind::ExecutionFailed { failed_step, .. } => { // TODO: obviously we shouldn't stop here, get past more of the - // update process in this test. We currently fail when attempting to - // look up the SP's board in our tuf repo. + // update process in this test. assert_eq!(failed_step.info.component, UpdateComponent::Sp); } other => { diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 36c3fe3f5f..6b40b98db6 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -32,18 +32,18 @@ const-oid = { version = "0.9.5", default-features = false, features = ["db", "st crossbeam-epoch = { version = "0.9.15" } crossbeam-utils = { version = "0.8.16" } crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } -diesel = { version = "2.1.1", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } +diesel = { version = "2.1.3", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0" } -flate2 = { version = "1.0.27" } -futures = { version = "0.3.28" } -futures-channel = { version = "0.3.28", features = ["sink"] } -futures-core = { version = "0.3.28" } -futures-io = { version = "0.3.28", default-features = false, features = ["std"] } -futures-sink = { version = "0.3.28" } -futures-task = { version = "0.3.28", default-features = false, features = ["std"] } -futures-util = { version = "0.3.28", features = ["channel", "io", "sink"] } -gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", features = ["std"] } +flate2 = { version = "1.0.28" } +futures = { version = "0.3.29" } +futures-channel = { version = "0.3.29", features = ["sink"] } +futures-core = { version = "0.3.29" } +futures-io = { version = "0.3.29", default-features = false, features = ["std"] } +futures-sink = { version = "0.3.29" } +futures-task = { version = "0.3.29", default-features = false, features = ["std"] } +futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } +gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } @@ -77,9 +77,10 @@ regex-syntax = { version = "0.7.5" } reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } -semver = { version = "1.0.18", features = ["serde"] } +semver = { version = "1.0.20", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } -sha2 = { version = "0.10.7", features = ["oid"] } +serde_json = { version = "1.0.107", features = ["raw_value"] } +sha2 = { version = "0.10.8", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } @@ -88,9 +89,8 @@ string_cache = { version = "0.8.7" } subtle = { version = "2.5.0" } syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } -textwrap = { version = "0.16.0" } time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } -tokio = { version = "1.32.0", features = ["full", "test-util"] } +tokio = { version = "1.33.0", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } @@ -124,18 +124,18 @@ const-oid = { version = "0.9.5", default-features = false, features = ["db", "st crossbeam-epoch = { version = "0.9.15" } crossbeam-utils = { version = "0.8.16" } crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } -diesel = { version = "2.1.1", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } +diesel = { version = "2.1.3", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0" } -flate2 = { version = "1.0.27" } -futures = { version = "0.3.28" } -futures-channel = { version = "0.3.28", features = ["sink"] } -futures-core = { version = "0.3.28" } -futures-io = { version = "0.3.28", default-features = false, features = ["std"] } -futures-sink = { version = "0.3.28" } -futures-task = { version = "0.3.28", default-features = false, features = ["std"] } -futures-util = { version = "0.3.28", features = ["channel", "io", "sink"] } -gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "1e180ae55e56bd17af35cb868ffbd18ce487351d", features = ["std"] } +flate2 = { version = "1.0.28" } +futures = { version = "0.3.29" } +futures-channel = { version = "0.3.29", features = ["sink"] } +futures-core = { version = "0.3.29" } +futures-io = { version = "0.3.29", default-features = false, features = ["std"] } +futures-sink = { version = "0.3.29" } +futures-task = { version = "0.3.29", default-features = false, features = ["std"] } +futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } +gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } @@ -169,9 +169,10 @@ regex-syntax = { version = "0.7.5" } reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } -semver = { version = "1.0.18", features = ["serde"] } +semver = { version = "1.0.20", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } -sha2 = { version = "0.10.7", features = ["oid"] } +serde_json = { version = "1.0.107", features = ["raw_value"] } +sha2 = { version = "0.10.8", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } @@ -180,10 +181,9 @@ string_cache = { version = "0.8.7" } subtle = { version = "2.5.0" } syn-dff4ba8e3ae991db = { package = "syn", version = "1.0.109", features = ["extra-traits", "fold", "full", "visit"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2.0.32", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } -textwrap = { version = "0.16.0" } time = { version = "0.3.27", features = ["formatting", "local-offset", "macros", "parsing"] } time-macros = { version = "0.2.13", default-features = false, features = ["formatting", "parsing"] } -tokio = { version = "1.32.0", features = ["full", "test-util"] } +tokio = { version = "1.33.0", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } @@ -200,49 +200,49 @@ zip = { version = "0.6.6", default-features = false, features = ["bzip2", "defla [target.x86_64-unknown-linux-gnu.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24.1" } +hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-unknown-linux-gnu.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24.1" } +hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-apple-darwin.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24.1" } +hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-apple-darwin.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24.1" } +hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.aarch64-apple-darwin.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24.1" } +hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.aarch64-apple-darwin.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24.1" } +hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } [target.x86_64-unknown-illumos.dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24.1" } +hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } @@ -251,7 +251,7 @@ toml_edit = { version = "0.19.15", features = ["serde"] } [target.x86_64-unknown-illumos.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } -hyper-rustls = { version = "0.24.1" } +hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] }