From 76106b11c08949563507cc0a86bd6bde520457e2 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 23 Apr 2024 12:17:18 +0100 Subject: [PATCH 01/39] Clear XDE underlay when destroying virtual hardware OPTE now prevents itself from being unloaded if its underlay state is set. Currently, underlay setup is performed only once, and it seems to be the case that XDE can be unloaded in some scenarios (e.g., `a4x2` setup). However, a consequence is that removing the driver requires an extra operation to explicitly clear the underlay state. This PR adds this operation to the `cargo xtask virtual-hardware destroy` command. This is currently blocked on opte#485 being approved/merged. Closes #5314. --- Cargo.lock | 14 +++++++------- Cargo.toml | 4 ++-- dev-tools/xtask/src/virtual_hardware.rs | 12 +++++++++++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f5477f077..46c1f4de60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1750,7 +1750,7 @@ dependencies = [ [[package]] name = "derror-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "darling", "proc-macro2", @@ -3467,7 +3467,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" [[package]] name = "illumos-utils" @@ -3874,7 +3874,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "quote", "syn 2.0.59", @@ -5953,7 +5953,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "cfg-if", "derror-macro", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -5983,7 +5983,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -6057,7 +6057,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index b6b937614c..df14c579d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -305,14 +305,14 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb", features = [ "api", "std" ] } once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb" } oso = "0.27" owo-colors = "4.0.0" oximeter = { path = "oximeter/oximeter" } diff --git a/dev-tools/xtask/src/virtual_hardware.rs b/dev-tools/xtask/src/virtual_hardware.rs index 95190ebfde..c480dc355c 100644 --- a/dev-tools/xtask/src/virtual_hardware.rs +++ b/dev-tools/xtask/src/virtual_hardware.rs @@ -100,6 +100,7 @@ const IPADM: &'static str = "/usr/sbin/ipadm"; const MODINFO: &'static str = "/usr/sbin/modinfo"; const MODUNLOAD: &'static str = "/usr/sbin/modunload"; const NETSTAT: &'static str = "/usr/bin/netstat"; +const OPTEADM: &'static str = "/opt/oxide/opte/bin/opteadm"; const PFEXEC: &'static str = "/usr/bin/pfexec"; const PING: &'static str = "/usr/sbin/ping"; const SWAP: &'static str = "/usr/sbin/swap"; @@ -240,8 +241,17 @@ fn unload_xde_driver() -> Result<()> { println!("xde driver already unloaded"); return Ok(()); }; - println!("unloading xde driver"); + println!("unloading xde driver:\na) clearing underlay..."); + let mut cmd = Command::new(PFEXEC); + cmd.args([OPTEADM, "clear-xde-underlay"]); + if let Err(e) = execute(cmd) { + // This is explicitly non-fatal: the underlay is only set when + // sled-agent is running. We still need to be able to tear + // down the driver if we immediately call create->destroy. + println!("\tFailed or already unset: {e}"); + } + println!("b) unloading module..."); let mut cmd = Command::new(PFEXEC); cmd.arg(MODUNLOAD); cmd.arg("-i"); From 94e180808689303d5a9896a347ea858766701e46 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 21 May 2024 12:27:27 +0100 Subject: [PATCH 02/39] Pull in latest OPTE after merge. --- .github/buildomat/jobs/a4x2-deploy.sh | 2 +- .github/buildomat/jobs/deploy.sh | 2 +- Cargo.lock | 28 ++++++++------------------- Cargo.toml | 8 ++++---- package-manifest.toml | 16 +++++++-------- tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 4 ++-- tools/maghemite_mgd_checksums | 4 ++-- tools/opte_version | 2 +- 9 files changed, 28 insertions(+), 40 deletions(-) diff --git a/.github/buildomat/jobs/a4x2-deploy.sh b/.github/buildomat/jobs/a4x2-deploy.sh index 323b3e2e28..53153beafb 100755 --- a/.github/buildomat/jobs/a4x2-deploy.sh +++ b/.github/buildomat/jobs/a4x2-deploy.sh @@ -2,7 +2,7 @@ #: #: name = "a4x2-deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.27" +#: target = "lab-2.0-opte-0.31" #: output_rules = [ #: "/out/falcon/*.log", #: "/out/falcon/*.err", diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index c947a05e10..bee2700d89 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -2,7 +2,7 @@ #: #: name = "helios / deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.28" +#: target = "lab-2.0-opte-0.31" #: output_rules = [ #: "%/var/svc/log/oxide-sled-agent:default.log*", #: "%/zone/oxz_*/root/var/svc/log/oxide-*.log*", diff --git a/Cargo.lock b/Cargo.lock index 2ec770ff83..ad3ac965d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1591,7 +1591,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=025389ff39d594bf2b815377e2c1dc4dd23b1f96#025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source = "git+https://github.com/oxidecomputer/maghemite?rev=a557af774b2c3cfe1e9e4e27de9ad85de6a02a98#a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" dependencies = [ "percent-encoding", "progenitor", @@ -1729,17 +1729,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derror-macro" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.64", -] - [[package]] name = "dhcproto" version = "0.11.0" @@ -3481,7 +3470,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" [[package]] name = "illumos-utils" @@ -3894,7 +3883,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "quote", "syn 2.0.64", @@ -4306,7 +4295,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=025389ff39d594bf2b815377e2c1dc4dd23b1f96#025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source = "git+https://github.com/oxidecomputer/maghemite?rev=a557af774b2c3cfe1e9e4e27de9ad85de6a02a98#a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" dependencies = [ "anyhow", "chrono", @@ -6019,10 +6008,9 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "cfg-if", - "derror-macro", "dyn-clone", "illumos-sys-hdrs", "kstat-macro", @@ -6037,7 +6025,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -6049,7 +6037,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -6123,7 +6111,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 8d3c80f6c5..8d298d1feb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -312,8 +312,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.12" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" } multimap = "0.10.0" nexus-client = { path = "clients/nexus-client" } nexus-config = { path = "nexus-config" } @@ -347,14 +347,14 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504", features = [ "api", "std" ] } once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504" } oso = "0.27" owo-colors = "4.0.0" oximeter = { path = "oximeter/oximeter" } diff --git a/package-manifest.toml b/package-manifest.toml index 2bfc51d533..096cd6f011 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -533,10 +533,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" # The SHA256 digest is automatically posted to: -# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt -source.sha256 = "f2ee54b6a654daa1c1f817440317e9b11c5ddc71249df261bb5cfa0e6057dc24" +# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt +source.sha256 = "b7a88a985bb29c105fffa184c9826e3243e87d3063987b90b39e51abcd8b8526" output.type = "tarball" [package.mg-ddm] @@ -549,10 +549,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "bb98815f759f38abee9f5aea0978cd33e66e75079cc8c171036be21bf9049c96" +source.sha256 = "8d0556bcf83360df653f72e885cf28e628601f17137d9475a234b1020c6366ba" output.type = "zone" output.intermediate_only = true @@ -564,10 +564,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" # The SHA256 digest is automatically posted to: -# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "e0907de39ca9f8ab45d40d361a1dbeed4bd8e9b157f8d3d8fe0a4bc259d933bd" +# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt +source.sha256 = "91fb4c893240e6f781ab6c3d6f2af8c0e9378fd646d1e45233502e3a4d23cb1a" output.type = "zone" output.intermediate_only = true diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index c39c9690bb..6cbb69ad79 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="025389ff39d594bf2b815377e2c1dc4dd23b1f96" +COMMIT="a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" SHA2="004e873e4120aa26460271368485266b75b7f964e5ed4dbee8fb5db4519470d7" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 966e4de7fe..fe46171c0d 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="025389ff39d594bf2b815377e2c1dc4dd23b1f96" -SHA2="a5d2f275c99152711dec1df58fd49d459d3fcb8fbfc7a7f48f432be248d74639" +COMMIT="a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +SHA2="fdb33ee7425923560534672264008ef8948d227afce948ab704de092ad72157c" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index eeb873a424..70471ccf6d 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="e0907de39ca9f8ab45d40d361a1dbeed4bd8e9b157f8d3d8fe0a4bc259d933bd" -MGD_LINUX_SHA256="903413ddaab89594ed7518cb8f2f27793e96cd17ed2d6b3fe11657ec4375cb19" +CIDL_SHA256="91fb4c893240e6f781ab6c3d6f2af8c0e9378fd646d1e45233502e3a4d23cb1a" +MGD_LINUX_SHA256="111c111691bb42e5fde4d5a3d2a613ffc7e7e57c536866f4f00fb5afb60c6da4" diff --git a/tools/opte_version b/tools/opte_version index e1b3e11499..fc3e603c41 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.28.233 +0.31.258 From b67f416773f69306920539ac6104c2ace0c6ee82 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 21 May 2024 12:55:18 +0100 Subject: [PATCH 03/39] Bump OPTE and related. --- Cargo.lock | 12 ++++++------ Cargo.toml | 4 ++-- package-manifest.toml | 12 ++++++------ tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 2 +- tools/maghemite_mgd_checksums | 4 ++-- tools/opte_version | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad3ac965d0..728722907e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3470,7 +3470,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" [[package]] name = "illumos-utils" @@ -3883,7 +3883,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "quote", "syn 2.0.64", @@ -6008,7 +6008,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "cfg-if", "dyn-clone", @@ -6025,7 +6025,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -6037,7 +6037,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -6111,7 +6111,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 8d298d1feb..b70fbe25c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -347,14 +347,14 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "a0294dbf886bcaf76d33b5c8f0ca6616a9b90781", features = [ "api", "std" ] } once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" } oso = "0.27" owo-colors = "4.0.0" oximeter = { path = "oximeter/oximeter" } diff --git a/package-manifest.toml b/package-manifest.toml index 096cd6f011..e86587245d 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -533,10 +533,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source.commit = "c9824727eedc66d4920e42e7260df05050841ab8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "b7a88a985bb29c105fffa184c9826e3243e87d3063987b90b39e51abcd8b8526" +source.sha256 = "4ba6bb87fe9bd2dcb69b27a218e0f9f10767fa8eb3c1439fbf0fa5f5a1921dd9" output.type = "tarball" [package.mg-ddm] @@ -549,10 +549,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source.commit = "c9824727eedc66d4920e42e7260df05050841ab8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "8d0556bcf83360df653f72e885cf28e628601f17137d9475a234b1020c6366ba" +source.sha256 = "45918b2ef4fe2be048cc8934155335a1b20e91fea6d818c385fe3964a3e8fdec" output.type = "zone" output.intermediate_only = true @@ -564,10 +564,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source.commit = "c9824727eedc66d4920e42e7260df05050841ab8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "91fb4c893240e6f781ab6c3d6f2af8c0e9378fd646d1e45233502e3a4d23cb1a" +source.sha256 = "6ae4bc3b332e91706c1c6633a7fc218aac65b7feff5643ee2dbbe79b841e0df3" output.type = "zone" output.intermediate_only = true diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 6cbb69ad79..edd1b0d670 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +COMMIT="c9824727eedc66d4920e42e7260df05050841ab8" SHA2="004e873e4120aa26460271368485266b75b7f964e5ed4dbee8fb5db4519470d7" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index fe46171c0d..efe080571c 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +COMMIT="c9824727eedc66d4920e42e7260df05050841ab8" SHA2="fdb33ee7425923560534672264008ef8948d227afce948ab704de092ad72157c" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 70471ccf6d..d2ad05383d 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="91fb4c893240e6f781ab6c3d6f2af8c0e9378fd646d1e45233502e3a4d23cb1a" -MGD_LINUX_SHA256="111c111691bb42e5fde4d5a3d2a613ffc7e7e57c536866f4f00fb5afb60c6da4" +CIDL_SHA256="6ae4bc3b332e91706c1c6633a7fc218aac65b7feff5643ee2dbbe79b841e0df3" +MGD_LINUX_SHA256="7930008cf8ce535a8b31043fc3edde0e825bd54d75f73234929bd0037ecc3a41" diff --git a/tools/opte_version b/tools/opte_version index fc3e603c41..a8f4dc593f 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.31.258 +0.32.262 From 5f7cfa8a49205ffb4fa86ffd7288054787f6ffe1 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 21 May 2024 13:02:41 +0100 Subject: [PATCH 04/39] Compat with newer OPTE, more believable router rules --- common/src/api/external/mod.rs | 3 + illumos-utils/src/opte/port_manager.rs | 9 ++ nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/schema_versions.rs | 3 +- nexus/db-model/src/vpc_subnet.rs | 3 + nexus/db-queries/src/db/datastore/vpc.rs | 98 +++++++++---- nexus/db-queries/src/db/fixed_data/vpc.rs | 18 ++- .../src/db/fixed_data/vpc_subnet.rs | 18 +++ nexus/src/app/sagas/vpc_create.rs | 133 ++++++++++++++++-- nexus/src/external_api/http_entrypoints.rs | 12 +- nexus/types/src/external_api/views.rs | 5 +- schema/crdb/dbinit.sql | 7 +- schema/crdb/vpc-subnet-routing/up01.sql | 3 + 13 files changed, 251 insertions(+), 62 deletions(-) create mode 100644 schema/crdb/vpc-subnet-routing/up01.sql diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 1c01782cc6..f385c0b4fa 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1680,6 +1680,9 @@ pub enum RouteTarget { #[display("inetgw:{0}")] /// Forward traffic to an internet gateway InternetGateway(Name), + #[display("drop")] + /// Drop matching traffic + Drop, } /// A `RouteDestination` is used to match traffic with a routing rule, on the diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 03c51c321d..d6b4963607 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -25,6 +25,7 @@ use oxide_vpc::api::IpCidr; use oxide_vpc::api::Ipv4Cfg; use oxide_vpc::api::Ipv6Cfg; use oxide_vpc::api::MacAddr; +use oxide_vpc::api::RouterClass; use oxide_vpc::api::RouterTarget; use oxide_vpc::api::SNat4Cfg; use oxide_vpc::api::SNat6Cfg; @@ -339,9 +340,16 @@ impl PortManager { (port, ticket) }; + // TODO: These should not be filled in like this, and should be informed + // by either our existing knowledge of current knowledge of system + custom + // routers OR we just await the router RPW filling this in for us. + // In future, ∃ VPCs *without* an Internet Gateway so we can't just + // plumb that in as well... + // Add a router entry for this interface's subnet, directing traffic to the // VPC subnet. let route = AddRouterEntryReq { + class: RouterClass::System, port_name: port_name.clone(), dest: vpc_subnet, target: RouterTarget::VpcSubnet(vpc_subnet), @@ -378,6 +386,7 @@ impl PortManager { .parse() .unwrap(); let route = AddRouterEntryReq { + class: RouterClass::System, port_name: port_name.clone(), dest, target: RouterTarget::InternetGateway, diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 224c461da0..95d372167e 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1099,6 +1099,7 @@ table! { rcgen -> Int8, ipv4_block -> Inet, ipv6_block -> Inet, + custom_router_id -> Nullable, } } diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index cb229274fe..47a5689d07 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(63, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(64, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(64, "vpc-subnet-routing"), KnownVersion::new(63, "remove-producer-base-route-column"), KnownVersion::new(62, "allocate-subnet-decommissioned-sleds"), KnownVersion::new(61, "blueprint-add-sled-state"), diff --git a/nexus/db-model/src/vpc_subnet.rs b/nexus/db-model/src/vpc_subnet.rs index 407c933ef2..5c85c8a6dc 100644 --- a/nexus/db-model/src/vpc_subnet.rs +++ b/nexus/db-model/src/vpc_subnet.rs @@ -39,6 +39,7 @@ pub struct VpcSubnet { pub rcgen: Generation, pub ipv4_block: Ipv4Net, pub ipv6_block: Ipv6Net, + pub custom_router_id: Option, } impl VpcSubnet { @@ -60,6 +61,7 @@ impl VpcSubnet { rcgen: Generation::new(), ipv4_block: Ipv4Net(ipv4_block), ipv6_block: Ipv6Net(ipv6_block), + custom_router_id: None, } } @@ -102,6 +104,7 @@ impl From for views::VpcSubnet { vpc_id: subnet.vpc_id, ipv4_block: subnet.ipv4_block.0, ipv6_block: subnet.ipv6_block.0, + custom_router_id: subnet.custom_router_id, } } } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 91843abf2e..d0c4e381ad 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -74,7 +74,8 @@ impl DataStore { ) -> Result<(), Error> { use crate::db::fixed_data::project::SERVICES_PROJECT_ID; use crate::db::fixed_data::vpc::SERVICES_VPC; - use crate::db::fixed_data::vpc::SERVICES_VPC_DEFAULT_ROUTE_ID; + use crate::db::fixed_data::vpc::SERVICES_VPC_DEFAULT_V4_ROUTE_ID; + use crate::db::fixed_data::vpc::SERVICES_VPC_DEFAULT_V6_ROUTE_ID; opctx.authorize(authz::Action::Modify, &authz::DATABASE).await?; @@ -135,35 +136,41 @@ impl DataStore { .map(|(authz_router, _)| authz_router)? }; - let route = RouterRoute::new( - *SERVICES_VPC_DEFAULT_ROUTE_ID, - SERVICES_VPC.system_router_id, - RouterRouteKind::Default, - nexus_types::external_api::params::RouterRouteCreate { - identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), - description: - "Default internet gateway route for Oxide Services" - .to_string(), + // Unwrap safety: these are known valid CIDR blocks. + let default_ips = [ + ("0.0.0.0/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V4_ROUTE_ID), + ("::/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V6_ROUTE_ID), + ]; + + for (default, uuid) in default_ips { + let route = RouterRoute::new( + uuid, + SERVICES_VPC.system_router_id, + RouterRouteKind::Default, + nexus_types::external_api::params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: + "Default internet gateway route for Oxide Services" + .to_string(), + }, + target: RouteTarget::InternetGateway( + "outbound".parse().unwrap(), + ), + destination: RouteDestination::IpNet(default), }, - target: RouteTarget::InternetGateway( - "outbound".parse().unwrap(), - ), - destination: RouteDestination::Vpc( - SERVICES_VPC.identity.name.clone().into(), - ), - }, - ); - self.router_create_route(opctx, &authz_router, route) - .await - .map(|_| ()) - .or_else(|e| match e { - Error::ObjectAlreadyExists { .. } => Ok(()), - _ => Err(e), - })?; + ); + self.router_create_route(opctx, &authz_router, route) + .await + .map(|_| ()) + .or_else(|e| match e { + Error::ObjectAlreadyExists { .. } => Ok(()), + _ => Err(e), + })?; + } self.load_builtin_vpc_fw_rules(opctx).await?; - self.load_builtin_vpc_subnets(opctx).await?; + self.load_builtin_vpc_subnets(opctx, &authz_router).await?; info!(opctx.log, "created built-in services vpc"); @@ -228,10 +235,14 @@ impl DataStore { async fn load_builtin_vpc_subnets( &self, opctx: &OpContext, + authz_router: &authz::VpcRouter, ) -> Result<(), Error> { use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET; + use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET_ROUTE_ID; use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; + use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET_ROUTE_ID; use crate::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET; + use crate::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET_ROUTE_ID; debug!(opctx.log, "attempting to create built-in VPC Subnets"); @@ -242,9 +253,11 @@ impl DataStore { .lookup_for(authz::Action::CreateChild) .await .internal_context("lookup built-in services vpc")?; - for vpc_subnet in - [&*DNS_VPC_SUBNET, &*NEXUS_VPC_SUBNET, &*NTP_VPC_SUBNET] - { + for (vpc_subnet, route_id) in [ + (&*DNS_VPC_SUBNET, *DNS_VPC_SUBNET_ROUTE_ID), + (&*NEXUS_VPC_SUBNET, *NEXUS_VPC_SUBNET_ROUTE_ID), + (&*NTP_VPC_SUBNET, *NTP_VPC_SUBNET_ROUTE_ID), + ] { if let Ok(_) = db::lookup::LookupPath::new(opctx, self) .vpc_subnet_id(vpc_subnet.id()) .fetch() @@ -260,6 +273,31 @@ impl DataStore { Error::ObjectAlreadyExists { .. } => Ok(()), _ => Err(e), })?; + + let route = RouterRoute::new( + route_id, + *SERVICES_VPC_ID, + RouterRouteKind::Default, + nexus_types::external_api::params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: + "Default internet gateway route for Oxide Services" + .to_string(), + }, + target: RouteTarget::Subnet(vpc_subnet.name().clone()), + destination: RouteDestination::Subnet( + vpc_subnet.name().clone(), + ), + }, + ); + self.router_create_route(opctx, &authz_router, route) + .await + .map(|_| ()) + .or_else(|e| match e { + Error::ObjectAlreadyExists { .. } => Ok(()), + _ => Err(e), + })?; } info!(opctx.log, "created built-in services vpc subnets"); diff --git a/nexus/db-queries/src/db/fixed_data/vpc.rs b/nexus/db-queries/src/db/fixed_data/vpc.rs index c71b655ddc..6dffc11426 100644 --- a/nexus/db-queries/src/db/fixed_data/vpc.rs +++ b/nexus/db-queries/src/db/fixed_data/vpc.rs @@ -24,11 +24,19 @@ pub static SERVICES_VPC_ROUTER_ID: Lazy = Lazy::new(|| { }); /// UUID of default route for built-in Services VPC. -pub static SERVICES_VPC_DEFAULT_ROUTE_ID: Lazy = Lazy::new(|| { - "001de000-074c-4000-8000-000000000002" - .parse() - .expect("invalid uuid for builtin services vpc default route id") -}); +pub static SERVICES_VPC_DEFAULT_V4_ROUTE_ID: Lazy = + Lazy::new(|| { + "001de000-074c-4000-8000-000000000002" + .parse() + .expect("invalid uuid for builtin services vpc default route id") + }); + +pub static SERVICES_VPC_DEFAULT_V6_ROUTE_ID: Lazy = + Lazy::new(|| { + "001de000-074c-4000-8000-000000000003" + .parse() + .expect("invalid uuid for builtin services vpc default route id") + }); /// Built-in VPC for internal services on the rack. pub static SERVICES_VPC: Lazy = Lazy::new(|| { diff --git a/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs b/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs index c42d4121c9..45db9b7e0b 100644 --- a/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs +++ b/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs @@ -31,6 +31,24 @@ pub static NTP_VPC_SUBNET_ID: Lazy = Lazy::new(|| { .expect("invalid uuid for builtin boundary ntp vpc subnet id") }); +pub static DNS_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { + "001de000-c470-4000-8000-000000000004" + .parse() + .expect("invalid uuid for builtin services vpc default route id") +}); + +pub static NEXUS_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { + "001de000-c470-4000-8000-000000000005" + .parse() + .expect("invalid uuid for builtin services vpc default route id") +}); + +pub static NTP_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { + "001de000-c470-4000-8000-000000000006" + .parse() + .expect("invalid uuid for builtin services vpc default route id") +}); + /// Built-in VPC Subnet for External DNS. pub static DNS_VPC_SUBNET: Lazy = Lazy::new(|| { VpcSubnet::new( diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index fdd117b850..647ee05da5 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -13,6 +13,7 @@ use nexus_db_queries::{authn, authz, db}; use nexus_defaults as defaults; use omicron_common::api::external; use omicron_common::api::external::IdentityMetadataCreateParams; +use omicron_common::api::external::IpNet; use omicron_common::api::external::LookupType; use omicron_common::api::external::RouteDestination; use omicron_common::api::external::RouteTarget; @@ -44,14 +45,22 @@ declare_saga_actions! { + svc_create_router - svc_create_router_undo } - VPC_CREATE_ROUTE -> "route" { - + svc_create_route - - svc_create_route_undo + VPC_CREATE_V4_ROUTE -> "route4" { + + svc_create_v4_route + - svc_create_v4_route_undo + } + VPC_CREATE_V6_ROUTE -> "route6" { + + svc_create_v6_route + - svc_create_v6_route_undo } VPC_CREATE_SUBNET -> "subnet" { + svc_create_subnet - svc_create_subnet_undo } + VPC_CREATE_SUBNET_ROUTE -> "route" { + + svc_create_subnet_route + - svc_create_subnet_route_undo + } VPC_UPDATE_FIREWALL -> "firewall" { + svc_update_firewall - svc_update_firewall_undo @@ -79,8 +88,18 @@ pub fn create_dag( ACTION_GENERATE_ID.as_ref(), )); builder.append(Node::action( - "default_route_id", - "GenerateDefaultRouteId", + "default_v4_route_id", + "GenerateDefaultV4RouteId", + ACTION_GENERATE_ID.as_ref(), + )); + builder.append(Node::action( + "default_v6_route_id", + "GenerateDefaultV6RouteId", + ACTION_GENERATE_ID.as_ref(), + )); + builder.append(Node::action( + "default_subnet_route_id", + "GenerateDefaultV6RouteId", ACTION_GENERATE_ID.as_ref(), )); builder.append(Node::action( @@ -90,8 +109,10 @@ pub fn create_dag( )); builder.append(vpc_create_vpc_action()); builder.append(vpc_create_router_action()); - builder.append(vpc_create_route_action()); + builder.append(vpc_create_v4_route_action()); + builder.append(vpc_create_v6_route_action()); builder.append(vpc_create_subnet_action()); + builder.append(vpc_create_subnet_route_action()); builder.append(vpc_update_firewall_action()); builder.append(vpc_notify_sleds_action()); @@ -217,8 +238,44 @@ async fn svc_create_router_undo( Ok(()) } +// XX: possibly do these as a subsaga? + +async fn svc_create_v4_route( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let default_route_id = sagactx.lookup::("default_v4_route_id")?; + let default_route = + "0.0.0.0/0".parse().expect("known-valid specifier for a default route"); + svc_create_route(sagactx, default_route_id, default_route).await +} + +async fn svc_create_v4_route_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let route_id = sagactx.lookup::("default_v4_route_id")?; + svc_create_route_undo(sagactx, route_id).await +} + +async fn svc_create_v6_route( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let default_route_id = sagactx.lookup::("default_v6_route_id")?; + let default_route = + "::/0".parse().expect("known-valid specifier for a default route"); + svc_create_route(sagactx, default_route_id, default_route).await +} + +async fn svc_create_v6_route_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let route_id = sagactx.lookup::("default_v6_route_id")?; + svc_create_route_undo(sagactx, route_id).await +} + async fn svc_create_route( sagactx: NexusActionContext, + route_id: Uuid, + default_net: IpNet, ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; @@ -226,12 +283,11 @@ async fn svc_create_route( &sagactx, ¶ms.serialized_authn, ); - let default_route_id = sagactx.lookup::("default_route_id")?; let system_router_id = sagactx.lookup::("system_router_id")?; let authz_router = sagactx.lookup::("router")?; let route = db::model::RouterRoute::new( - default_route_id, + route_id, system_router_id, RouterRouteKind::Default, params::RouterRouteCreate { @@ -240,9 +296,7 @@ async fn svc_create_route( description: "The default route of a vpc".to_string(), }, target: RouteTarget::InternetGateway("outbound".parse().unwrap()), - destination: RouteDestination::Vpc( - params.vpc_create.identity.name.clone(), - ), + destination: RouteDestination::IpNet(default_net), }, ); @@ -256,6 +310,7 @@ async fn svc_create_route( async fn svc_create_route_undo( sagactx: NexusActionContext, + route_id: Uuid, ) -> Result<(), anyhow::Error> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; @@ -264,7 +319,6 @@ async fn svc_create_route_undo( ¶ms.serialized_authn, ); let authz_router = sagactx.lookup::("router")?; - let route_id = sagactx.lookup::("default_route_id")?; let authz_route = authz::RouterRoute::new( authz_router, route_id, @@ -370,6 +424,61 @@ async fn svc_create_subnet_undo( Ok(()) } +async fn svc_create_subnet_route( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let system_router_id = sagactx.lookup::("system_router_id")?; + let authz_router = sagactx.lookup::("router")?; + let route_id = sagactx.lookup::("default_subnet_route_id")?; + + let route = db::model::RouterRoute::new( + route_id, + system_router_id, + RouterRouteKind::Default, + params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: "The default route of a vpc".to_string(), + }, + target: RouteTarget::Subnet("default".parse().unwrap()), + destination: RouteDestination::Subnet("default".parse().unwrap()), + }, + ); + + osagactx + .datastore() + .router_create_route(&opctx, &authz_router, route) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + +async fn svc_create_subnet_route_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let authz_router = sagactx.lookup::("router")?; + let route_id = sagactx.lookup::("default_subnet_route_id")?; + let authz_route = authz::RouterRoute::new( + authz_router, + route_id, + LookupType::ById(route_id), + ); + osagactx.datastore().router_delete_route(&opctx, &authz_route).await?; + Ok(()) +} + async fn svc_update_firewall( sagactx: NexusActionContext, ) -> Result, ActionError> { diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 350836441e..2678768b48 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -5446,7 +5446,6 @@ async fn vpc_firewall_rules_update( method = GET, path = "/v1/vpc-routers", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_list( rqctx: RequestContext, @@ -5486,7 +5485,6 @@ async fn vpc_router_list( method = GET, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_view( rqctx: RequestContext, @@ -5520,7 +5518,6 @@ async fn vpc_router_view( method = POST, path = "/v1/vpc-routers", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_create( rqctx: RequestContext, @@ -5556,7 +5553,6 @@ async fn vpc_router_create( method = DELETE, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_delete( rqctx: RequestContext, @@ -5590,7 +5586,6 @@ async fn vpc_router_delete( method = PUT, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_update( rqctx: RequestContext, @@ -5630,7 +5625,6 @@ async fn vpc_router_update( method = GET, path = "/v1/vpc-router-routes", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_list( rqctx: RequestContext, @@ -5672,7 +5666,6 @@ async fn vpc_router_route_list( method = GET, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_view( rqctx: RequestContext, @@ -5704,12 +5697,11 @@ async fn vpc_router_route_view( .await } -/// Create router +/// Create route #[endpoint { method = POST, path = "/v1/vpc-router-routes", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_create( rqctx: RequestContext, @@ -5745,7 +5737,6 @@ async fn vpc_router_route_create( method = DELETE, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_delete( rqctx: RequestContext, @@ -5781,7 +5772,6 @@ async fn vpc_router_route_delete( method = PUT, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_update( rqctx: RequestContext, diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 1e90d04b55..cb05f85930 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -258,7 +258,7 @@ pub struct Vpc { } /// A VPC subnet represents a logical grouping for instances that allows network traffic between -/// them, within a IPv4 subnetwork or optionall an IPv6 subnetwork. +/// them, within a IPv4 subnetwork or optionally an IPv6 subnetwork. #[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct VpcSubnet { /// common identifying metadata @@ -273,6 +273,9 @@ pub struct VpcSubnet { /// The IPv6 subnet CIDR block. pub ipv6_block: Ipv6Net, + + /// ID for an attached custom router. + pub custom_router_id: Option, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index cc298e4565..486ae68240 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -1367,7 +1367,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.vpc_subnet ( /* Child resource creation generation number */ rcgen INT8 NOT NULL, ipv4_block INET NOT NULL, - ipv6_block INET NOT NULL + ipv6_block INET NOT NULL, + /* nullable FK to the `vpc_router` table. */ + custom_router_id UUID ); /* Subnet and network interface names are unique per VPC, not project */ @@ -1623,6 +1625,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.router_route ( /* Indicates that the object has been deleted */ time_deleted TIMESTAMPTZ, + /* FK to the `vpc_router` table. */ vpc_router_id UUID NOT NULL, kind omicron.public.router_route_kind NOT NULL, target STRING(128) NOT NULL, @@ -3859,7 +3862,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '63.0.0', NULL) + (TRUE, NOW(), NOW(), '64.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/vpc-subnet-routing/up01.sql b/schema/crdb/vpc-subnet-routing/up01.sql new file mode 100644 index 0000000000..d1869dd010 --- /dev/null +++ b/schema/crdb/vpc-subnet-routing/up01.sql @@ -0,0 +1,3 @@ +-- Each subnet may have a custom router attached. +ALTER TABLE omicron.public.vpc_subnet +ADD COLUMN IF NOT EXISTS custom_router_id UUID; From 16e0107f71774d3ef429f76f2174d99aeb77c9c5 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 15 May 2024 14:39:43 +0100 Subject: [PATCH 05/39] VPC Subnet route reconcile. --- nexus/db-model/src/vpc_route.rs | 23 +++- nexus/db-queries/src/db/datastore/vpc.rs | 167 ++++++++++++++++++++--- nexus/src/app/sagas/vpc_create.rs | 26 ++-- 3 files changed, 179 insertions(+), 37 deletions(-) diff --git a/nexus/db-model/src/vpc_route.rs b/nexus/db-model/src/vpc_route.rs index 168ed41cef..2c561325a5 100644 --- a/nexus/db-model/src/vpc_route.rs +++ b/nexus/db-model/src/vpc_route.rs @@ -18,7 +18,7 @@ use std::io::Write; use uuid::Uuid; impl_enum_wrapper!( - #[derive(SqlType, Debug)] + #[derive(SqlType, Debug, QueryId)] #[diesel(postgres_type(name = "router_route_kind", schema = "public"))] pub struct RouterRouteKindEnum; @@ -127,6 +127,27 @@ impl RouterRoute { destination: RouteDestination::new(params.destination), } } + + pub fn for_subnet( + route_id: Uuid, + system_router_id: Uuid, + subnet: Name, + ) -> Result { + let name = format!("subnet_{}", subnet).parse().map_err(|_| ())?; + Ok(Self::new( + route_id, + system_router_id, + external::RouterRouteKind::VpcSubnet, + params::RouterRouteCreate { + identity: external::IdentityMetadataCreateParams { + name, + description: format!("VPC Subnet route for '{subnet}'"), + }, + target: external::RouteTarget::Subnet(subnet.0.clone()), + destination: external::RouteDestination::Subnet(subnet.0), + }, + )) + } } impl Into for RouterRoute { diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index d0c4e381ad..029f46e3c1 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -21,6 +21,7 @@ use crate::db::model::InstanceNetworkInterface; use crate::db::model::Name; use crate::db::model::Project; use crate::db::model::RouterRoute; +use crate::db::model::RouterRouteKind; use crate::db::model::RouterRouteUpdate; use crate::db::model::Sled; use crate::db::model::Vni; @@ -59,11 +60,12 @@ use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_common::api::external::RouteDestination; use omicron_common::api::external::RouteTarget; -use omicron_common::api::external::RouterRouteKind; +use omicron_common::api::external::RouterRouteKind as ExternalRouteKind; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::Vni as ExternalVni; use ref_cast::RefCast; use std::collections::BTreeMap; +use std::collections::HashSet; use uuid::Uuid; impl DataStore { @@ -146,7 +148,7 @@ impl DataStore { let route = RouterRoute::new( uuid, SERVICES_VPC.system_router_id, - RouterRouteKind::Default, + ExternalRouteKind::Default, nexus_types::external_api::params::RouterRouteCreate { identity: IdentityMetadataCreateParams { name: "default".parse().unwrap(), @@ -274,23 +276,12 @@ impl DataStore { _ => Err(e), })?; - let route = RouterRoute::new( + let route = RouterRoute::for_subnet( route_id, *SERVICES_VPC_ID, - RouterRouteKind::Default, - nexus_types::external_api::params::RouterRouteCreate { - identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), - description: - "Default internet gateway route for Oxide Services" - .to_string(), - }, - target: RouteTarget::Subnet(vpc_subnet.name().clone()), - destination: RouteDestination::Subnet( - vpc_subnet.name().clone(), - ), - }, - ); + vpc_subnet.name().clone().into(), + ) + .expect("builtin service names are short enough for route naming"); self.router_create_route(opctx, &authz_router, route) .await .map(|_| ()) @@ -808,6 +799,9 @@ impl DataStore { assert_eq!(authz_vpc.id(), subnet.vpc_id); let db_subnet = self.vpc_create_subnet_raw(subnet).await?; + self.vpc_system_router_ensure_subnet_routes(opctx, authz_vpc.id()) + .await + .map_err(SubnetError::External)?; Ok(( authz::VpcSubnet::new( authz_vpc.clone(), @@ -888,6 +882,12 @@ impl DataStore { "deletion failed due to concurrent modification", )); } else { + self.vpc_system_router_ensure_subnet_routes( + opctx, + db_subnet.vpc_id, + ) + .await?; + Ok(()) } } @@ -901,7 +901,7 @@ impl DataStore { opctx.authorize(authz::Action::Modify, authz_subnet).await?; use db::schema::vpc_subnet::dsl; - diesel::update(dsl::vpc_subnet) + let out = diesel::update(dsl::vpc_subnet) .filter(dsl::time_deleted.is_null()) .filter(dsl::id.eq(authz_subnet.id())) .set(updates) @@ -913,7 +913,11 @@ impl DataStore { e, ErrorHandler::NotFoundByResource(authz_subnet), ) - }) + })?; + + self.vpc_system_router_ensure_subnet_routes(opctx, out.vpc_id).await?; + + Ok(out) } pub async fn subnet_list_instance_network_interfaces( @@ -1097,6 +1101,17 @@ impl DataStore { assert_eq!(authz_router.id(), route.vpc_router_id); opctx.authorize(authz::Action::CreateChild, authz_router).await?; + Self::router_create_route_on_connection( + route, + &*self.pool_connection_authorized(opctx).await?, + ) + .await + } + + pub async fn router_create_route_on_connection( + route: RouterRoute, + conn: &async_bb8_diesel::Connection, + ) -> CreateResult { use db::schema::router_route::dsl; let router_id = route.vpc_router_id; let name = route.name().clone(); @@ -1105,9 +1120,7 @@ impl DataStore { router_id, diesel::insert_into(dsl::router_route).values(route), ) - .insert_and_get_result_async( - &*self.pool_connection_authorized(opctx).await?, - ) + .insert_and_get_result_async(conn) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { @@ -1259,6 +1272,116 @@ impl DataStore { ) }) } + + /// Ensure the system router for a VPC has the correct set of subnet + /// routing rules, after any changes to a subnet. + pub async fn vpc_system_router_ensure_subnet_routes( + &self, + opctx: &OpContext, + vpc_id: Uuid, + ) -> Result<(), Error> { + // These rules are immutable from a user's perspective, and + // aren't something which they can meaningfully interact with, + // so uuid stability on e.g. VPC rename is not a primary concern. + // We make sure only to alter VPC subnet rules here: users may + // modify other system routes like internet gateways. + let conn = self.pool_connection_authorized(opctx).await?; + let log = opctx.log.clone(); + self.transaction_retry_wrapper("vpc_subnet_route_reconcile") + .transaction(&conn, |conn| { + let log = log.clone(); + async move { + use db::schema::router_route::dsl; + use db::schema::vpc_subnet::dsl as subnet; + use db::schema::vpc::dsl as vpc; + + let system_router_id = vpc::vpc + .filter(vpc::id.eq(vpc_id)) + .filter(vpc::time_deleted.is_null()) + .select(vpc::system_router_id) + .limit(1) + .get_result_async(&conn) + .await?; + + let valid_subnets: Vec = subnet::vpc_subnet + .filter(subnet::id.eq(vpc_id)) + .filter(subnet::time_deleted.is_null()) + .select(VpcSubnet::as_select()) + .load_async(&conn) + .await?; + + let current_rules: Vec = dsl::router_route + .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::vpc_router_id.eq(system_router_id)) + .select(RouterRoute::as_select()) + .load_async(&conn) + .await?; + + // Build the add/delete sets. + let expected_names: HashSet = valid_subnets.iter() + .map(|v| v.identity.name.clone()) + .collect(); + + let mut found_names = HashSet::new(); + let mut invalid = Vec::new(); + for rule in current_rules { + let id = rule.id(); + match (rule.kind.0, rule.target.0) { + (ExternalRouteKind::VpcSubnet, RouteTarget::Subnet(n)) + if expected_names.contains(Name::ref_cast(&n)) => + {let _ = found_names.insert(n.into());}, + _ => invalid.push(id), + } + } + + // Add/Remove routes. Retry if numebr is incorrect due to + // concurrent modification. + let now = Utc::now(); + let to_update = invalid.len(); + let updated_rows = diesel::update(dsl::router_route) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq_any(invalid)) + .set(dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await?; + + if updated_rows != to_update { + return Err(DieselError::RollbackTransaction); + } + + // Duplicate rules are caught here using the UNIQUE constraint + // on names in a router. Only nexus can alter the system router, + // so there is no risk of collision with user-specified names. + for subnet in expected_names.difference(&found_names) { + let route_id = Uuid::new_v4(); + // XXX this is fallible as it is based on subnet name. + // need to control this somewhere sane. + let Ok(route) = db::model::RouterRoute::for_subnet( + route_id, + system_router_id, + subnet.clone(), + ) else { + error!( + log, + "Reconciling VPC routes: name {} in vpc {} is too long", + subnet, + vpc_id, + ); + continue; + }; + + match Self::router_create_route_on_connection(route, &conn).await { + Err(Error::Conflict { .. }) => return Err(DieselError::RollbackTransaction), + Err(_) => return Err(DieselError::NotFound), + _ => {}, + } + } + + Ok(()) + }}).await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } } #[cfg(test)] diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index 647ee05da5..475ff0bf58 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -246,7 +246,8 @@ async fn svc_create_v4_route( let default_route_id = sagactx.lookup::("default_v4_route_id")?; let default_route = "0.0.0.0/0".parse().expect("known-valid specifier for a default route"); - svc_create_route(sagactx, default_route_id, default_route).await + svc_create_route(sagactx, default_route_id, default_route, "default_v4") + .await } async fn svc_create_v4_route_undo( @@ -262,7 +263,8 @@ async fn svc_create_v6_route( let default_route_id = sagactx.lookup::("default_v6_route_id")?; let default_route = "::/0".parse().expect("known-valid specifier for a default route"); - svc_create_route(sagactx, default_route_id, default_route).await + svc_create_route(sagactx, default_route_id, default_route, "default_v6") + .await } async fn svc_create_v6_route_undo( @@ -276,6 +278,7 @@ async fn svc_create_route( sagactx: NexusActionContext, route_id: Uuid, default_net: IpNet, + name: &str, ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; @@ -292,7 +295,7 @@ async fn svc_create_route( RouterRouteKind::Default, params::RouterRouteCreate { identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), + name: name.parse().unwrap(), description: "The default route of a vpc".to_string(), }, target: RouteTarget::InternetGateway("outbound".parse().unwrap()), @@ -436,20 +439,15 @@ async fn svc_create_subnet_route( let system_router_id = sagactx.lookup::("system_router_id")?; let authz_router = sagactx.lookup::("router")?; let route_id = sagactx.lookup::("default_subnet_route_id")?; + let (_, db_subnet) = + sagactx.lookup::<(authz::VpcSubnet, db::model::VpcSubnet)>("subnet")?; - let route = db::model::RouterRoute::new( + let route = db::model::RouterRoute::for_subnet( route_id, system_router_id, - RouterRouteKind::Default, - params::RouterRouteCreate { - identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), - description: "The default route of a vpc".to_string(), - }, - target: RouteTarget::Subnet("default".parse().unwrap()), - destination: RouteDestination::Subnet("default".parse().unwrap()), - }, - ); + db_subnet.identity.name, + ) + .expect("default subnet name is short enough for route naming"); osagactx .datastore() From 22b71bb3c7a598f441e30aca2d4c30b4e29bdae7 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 15 May 2024 16:57:32 +0100 Subject: [PATCH 06/39] We now have valid, sane, default system routes These update in response to VPC subnet changes. Now to plumb them into OPTE. --- nexus/db-model/src/vpc_route.rs | 2 +- nexus/db-queries/src/db/datastore/vpc.rs | 31 ++++++++++-- nexus/src/app/sagas/vpc_create.rs | 64 +----------------------- nexus/src/app/vpc_router.rs | 1 - 4 files changed, 29 insertions(+), 69 deletions(-) diff --git a/nexus/db-model/src/vpc_route.rs b/nexus/db-model/src/vpc_route.rs index 2c561325a5..dda7f0b785 100644 --- a/nexus/db-model/src/vpc_route.rs +++ b/nexus/db-model/src/vpc_route.rs @@ -133,7 +133,7 @@ impl RouterRoute { system_router_id: Uuid, subnet: Name, ) -> Result { - let name = format!("subnet_{}", subnet).parse().map_err(|_| ())?; + let name = format!("sn-{}", subnet).parse().map_err(|_| ())?; Ok(Self::new( route_id, system_router_id, diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 029f46e3c1..d200f67663 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -140,18 +140,26 @@ impl DataStore { // Unwrap safety: these are known valid CIDR blocks. let default_ips = [ - ("0.0.0.0/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V4_ROUTE_ID), - ("::/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V6_ROUTE_ID), + ( + "default-v4", + "0.0.0.0/0".parse().unwrap(), + *SERVICES_VPC_DEFAULT_V4_ROUTE_ID, + ), + ( + "default-v6", + "::/0".parse().unwrap(), + *SERVICES_VPC_DEFAULT_V6_ROUTE_ID, + ), ]; - for (default, uuid) in default_ips { + for (name, default, uuid) in default_ips { let route = RouterRoute::new( uuid, SERVICES_VPC.system_router_id, ExternalRouteKind::Default, nexus_types::external_api::params::RouterRouteCreate { identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), + name: name.parse().unwrap(), description: "Default internet gateway route for Oxide Services" .to_string(), @@ -1036,6 +1044,18 @@ impl DataStore { ErrorHandler::NotFoundByResource(authz_router), ) })?; + + // All child routes are deleted. + use db::schema::router_route::dsl as rr; + let now = Utc::now(); + diesel::update(rr::router_route) + .filter(rr::time_deleted.is_null()) + .filter(rr::vpc_router_id.eq(authz_router.id())) + .set(rr::time_deleted.eq(now)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) } @@ -1291,6 +1311,7 @@ impl DataStore { .transaction(&conn, |conn| { let log = log.clone(); async move { + use db::schema::router_route::dsl; use db::schema::vpc_subnet::dsl as subnet; use db::schema::vpc::dsl as vpc; @@ -1304,7 +1325,7 @@ impl DataStore { .await?; let valid_subnets: Vec = subnet::vpc_subnet - .filter(subnet::id.eq(vpc_id)) + .filter(subnet::vpc_id.eq(vpc_id)) .filter(subnet::time_deleted.is_null()) .select(VpcSubnet::as_select()) .load_async(&conn) diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index 475ff0bf58..cc62d9315d 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -57,10 +57,6 @@ declare_saga_actions! { + svc_create_subnet - svc_create_subnet_undo } - VPC_CREATE_SUBNET_ROUTE -> "route" { - + svc_create_subnet_route - - svc_create_subnet_route_undo - } VPC_UPDATE_FIREWALL -> "firewall" { + svc_update_firewall - svc_update_firewall_undo @@ -97,11 +93,6 @@ pub fn create_dag( "GenerateDefaultV6RouteId", ACTION_GENERATE_ID.as_ref(), )); - builder.append(Node::action( - "default_subnet_route_id", - "GenerateDefaultV6RouteId", - ACTION_GENERATE_ID.as_ref(), - )); builder.append(Node::action( "default_subnet_id", "GenerateDefaultSubnetId", @@ -112,7 +103,6 @@ pub fn create_dag( builder.append(vpc_create_v4_route_action()); builder.append(vpc_create_v6_route_action()); builder.append(vpc_create_subnet_action()); - builder.append(vpc_create_subnet_route_action()); builder.append(vpc_update_firewall_action()); builder.append(vpc_notify_sleds_action()); @@ -246,7 +236,7 @@ async fn svc_create_v4_route( let default_route_id = sagactx.lookup::("default_v4_route_id")?; let default_route = "0.0.0.0/0".parse().expect("known-valid specifier for a default route"); - svc_create_route(sagactx, default_route_id, default_route, "default_v4") + svc_create_route(sagactx, default_route_id, default_route, "default-v4") .await } @@ -263,7 +253,7 @@ async fn svc_create_v6_route( let default_route_id = sagactx.lookup::("default_v6_route_id")?; let default_route = "::/0".parse().expect("known-valid specifier for a default route"); - svc_create_route(sagactx, default_route_id, default_route, "default_v6") + svc_create_route(sagactx, default_route_id, default_route, "default-v6") .await } @@ -427,56 +417,6 @@ async fn svc_create_subnet_undo( Ok(()) } -async fn svc_create_subnet_route( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let system_router_id = sagactx.lookup::("system_router_id")?; - let authz_router = sagactx.lookup::("router")?; - let route_id = sagactx.lookup::("default_subnet_route_id")?; - let (_, db_subnet) = - sagactx.lookup::<(authz::VpcSubnet, db::model::VpcSubnet)>("subnet")?; - - let route = db::model::RouterRoute::for_subnet( - route_id, - system_router_id, - db_subnet.identity.name, - ) - .expect("default subnet name is short enough for route naming"); - - osagactx - .datastore() - .router_create_route(&opctx, &authz_router, route) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - -async fn svc_create_subnet_route_undo( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let authz_router = sagactx.lookup::("router")?; - let route_id = sagactx.lookup::("default_subnet_route_id")?; - let authz_route = authz::RouterRoute::new( - authz_router, - route_id, - LookupType::ById(route_id), - ); - osagactx.datastore().router_delete_route(&opctx, &authz_route).await?; - Ok(()) -} - async fn svc_update_firewall( sagactx: NexusActionContext, ) -> Result, ActionError> { diff --git a/nexus/src/app/vpc_router.rs b/nexus/src/app/vpc_router.rs index 523a450bbd..e65b2a8605 100644 --- a/nexus/src/app/vpc_router.rs +++ b/nexus/src/app/vpc_router.rs @@ -114,7 +114,6 @@ impl super::Nexus { .await } - // TODO: When a router is deleted all its routes should be deleted // TODO: When a router is deleted it should be unassociated w/ any subnets it may be associated with // or trigger an error pub(crate) async fn vpc_delete_router( From 160853b2cb92a012b0124557a16d9f7e76ea08dc Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 15 May 2024 17:42:28 +0100 Subject: [PATCH 07/39] Start refreshing API specs --- nexus/tests/output/nexus_tags.txt | 10 + openapi/nexus.json | 1695 ++++++++++++++++++++++++----- 2 files changed, 1413 insertions(+), 292 deletions(-) diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index a32fe5c4b9..35d8c32561 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -232,6 +232,16 @@ vpc_delete DELETE /v1/vpcs/{vpc} vpc_firewall_rules_update PUT /v1/vpc-firewall-rules vpc_firewall_rules_view GET /v1/vpc-firewall-rules vpc_list GET /v1/vpcs +vpc_router_create POST /v1/vpc-routers +vpc_router_delete DELETE /v1/vpc-routers/{router} +vpc_router_list GET /v1/vpc-routers +vpc_router_route_create POST /v1/vpc-router-routes +vpc_router_route_delete DELETE /v1/vpc-router-routes/{route} +vpc_router_route_list GET /v1/vpc-router-routes +vpc_router_route_update PUT /v1/vpc-router-routes/{route} +vpc_router_route_view GET /v1/vpc-router-routes/{route} +vpc_router_update PUT /v1/vpc-routers/{router} +vpc_router_view GET /v1/vpc-routers/{router} vpc_subnet_create POST /v1/vpc-subnets vpc_subnet_delete DELETE /v1/vpc-subnets/{subnet} vpc_subnet_list GET /v1/vpc-subnets diff --git a/openapi/nexus.json b/openapi/nexus.json index 2bf6f0a6ff..55f83f4a24 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -8346,13 +8346,14 @@ } } }, - "/v1/vpc-subnets": { + "/v1/vpc-router-routes": { "get": { "tags": [ "vpcs" ], - "summary": "List subnets", - "operationId": "vpc_subnet_list", + "summary": "List routes", + "description": "List the routes associated with a router in a particular VPC.", + "operationId": "vpc_router_route_list", "parameters": [ { "in": "query", @@ -8382,6 +8383,14 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "sort_by", @@ -8392,7 +8401,7 @@ { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8404,7 +8413,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetResultsPage" + "$ref": "#/components/schemas/RouterRouteResultsPage" } } } @@ -8418,7 +8427,7 @@ }, "x-dropshot-pagination": { "required": [ - "vpc" + "router" ] } }, @@ -8426,8 +8435,8 @@ "tags": [ "vpcs" ], - "summary": "Create subnet", - "operationId": "vpc_subnet_create", + "summary": "Create route", + "operationId": "vpc_router_route_create", "parameters": [ { "in": "query", @@ -8439,19 +8448,27 @@ }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetCreate" + "$ref": "#/components/schemas/RouterRouteCreate" } } }, @@ -8463,7 +8480,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8477,18 +8494,18 @@ } } }, - "/v1/vpc-subnets/{subnet}": { + "/v1/vpc-router-routes/{route}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch subnet", - "operationId": "vpc_subnet_view", + "summary": "Fetch route", + "operationId": "vpc_router_route_view", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8502,10 +8519,19 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8517,7 +8543,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8534,13 +8560,13 @@ "tags": [ "vpcs" ], - "summary": "Update subnet", - "operationId": "vpc_subnet_update", + "summary": "Update route", + "operationId": "vpc_router_route_update", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8554,10 +8580,18 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8567,7 +8601,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetUpdate" + "$ref": "#/components/schemas/RouterRouteUpdate" } } }, @@ -8579,7 +8613,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8596,13 +8630,13 @@ "tags": [ "vpcs" ], - "summary": "Delete subnet", - "operationId": "vpc_subnet_delete", + "summary": "Delete route", + "operationId": "vpc_router_route_delete", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8616,10 +8650,18 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8638,23 +8680,14 @@ } } }, - "/v1/vpc-subnets/{subnet}/network-interfaces": { + "/v1/vpc-routers": { "get": { "tags": [ "vpcs" ], - "summary": "List network interfaces", - "operationId": "vpc_subnet_list_network_interfaces", + "summary": "List routers", + "operationId": "vpc_router_list", "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "limit", @@ -8705,7 +8738,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" + "$ref": "#/components/schemas/VpcRouterResultsPage" } } } @@ -8718,89 +8751,30 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "vpc" + ] } - } - }, - "/v1/vpcs": { - "get": { + }, + "post": { "tags": [ "vpcs" ], - "summary": "List VPCs", - "operationId": "vpc_list", + "summary": "Create VPC router", + "operationId": "vpc_router_create", "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "project" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create VPC", - "operationId": "vpc_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8811,7 +8785,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcCreate" + "$ref": "#/components/schemas/VpcRouterCreate" } } }, @@ -8823,7 +8797,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -8837,18 +8811,18 @@ } } }, - "/v1/vpcs/{vpc}": { + "/v1/vpc-routers/{router}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch VPC", - "operationId": "vpc_view", + "summary": "Fetch router", + "operationId": "vpc_router_view", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8857,7 +8831,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8869,7 +8851,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -8886,13 +8868,13 @@ "tags": [ "vpcs" ], - "summary": "Update a VPC", - "operationId": "vpc_update", + "summary": "Update router", + "operationId": "vpc_router_update", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8901,7 +8883,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8911,7 +8901,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcUpdate" + "$ref": "#/components/schemas/VpcRouterUpdate" } } }, @@ -8923,7 +8913,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -8940,13 +8930,13 @@ "tags": [ "vpcs" ], - "summary": "Delete VPC", - "operationId": "vpc_delete", + "summary": "Delete router", + "operationId": "vpc_router_delete", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8955,7 +8945,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8973,43 +8971,671 @@ } } } - } - }, - "components": { - "schemas": { - "Address": { - "description": "An address tied to an address lot.", - "type": "object", - "properties": { - "address": { - "description": "The address and prefix length of this address.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] + }, + "/v1/vpc-subnets": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List subnets", + "operationId": "vpc_subnet_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } }, - "address_lot": { - "description": "The address lot this address is drawn from.", - "allOf": [ - { - "$ref": "#/components/schemas/NameOrId" - } - ] - } - }, - "required": [ - "address", - "address_lot" - ] - }, - "AddressConfig": { - "description": "A set of addresses associated with a port configuration.", - "type": "object", - "properties": { - "addresses": { - "description": "The set of addresses assigned to the port configuration.", - "type": "array", + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnetResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "vpc" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create subnet", + "operationId": "vpc_subnet_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnetCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpc-subnets/{subnet}": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "Fetch subnet", + "operationId": "vpc_subnet_view", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "vpcs" + ], + "summary": "Update subnet", + "operationId": "vpc_subnet_update", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnetUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "vpcs" + ], + "summary": "Delete subnet", + "operationId": "vpc_subnet_delete", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpc-subnets/{subnet}/network-interfaces": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List network interfaces", + "operationId": "vpc_subnet_list_network_interfaces", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/vpcs": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List VPCs", + "operationId": "vpc_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create VPC", + "operationId": "vpc_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpcs/{vpc}": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "Fetch VPC", + "operationId": "vpc_view", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "vpcs" + ], + "summary": "Update a VPC", + "operationId": "vpc_update", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "vpcs" + ], + "summary": "Delete VPC", + "operationId": "vpc_delete", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Address": { + "description": "An address tied to an address lot.", + "type": "object", + "properties": { + "address": { + "description": "The address and prefix length of this address.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "address_lot": { + "description": "The address lot this address is drawn from.", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + } + }, + "required": [ + "address", + "address_lot" + ] + }, + "AddressConfig": { + "description": "A set of addresses associated with a port configuration.", + "type": "object", + "properties": { + "addresses": { + "description": "The set of addresses assigned to the port configuration.", + "type": "array", "items": { "$ref": "#/components/schemas/Address" } @@ -15617,22 +16243,169 @@ } }, "required": [ - "description", - "id", - "name", - "time_created", - "time_modified" + "description", + "id", + "name", + "time_created", + "time_modified" + ] + }, + "ProjectCreate": { + "description": "Create-time parameters for a `Project`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "description", + "name" + ] + }, + "ProjectResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Project" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "ProjectRole": { + "type": "string", + "enum": [ + "admin", + "collaborator", + "viewer" + ] + }, + "ProjectRolePolicy": { + "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", + "type": "object", + "properties": { + "role_assignments": { + "description": "Roles directly assigned on this resource", + "type": "array", + "items": { + "$ref": "#/components/schemas/ProjectRoleRoleAssignment" + } + } + }, + "required": [ + "role_assignments" + ] + }, + "ProjectRoleRoleAssignment": { + "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", + "type": "object", + "properties": { + "identity_id": { + "type": "string", + "format": "uuid" + }, + "identity_type": { + "$ref": "#/components/schemas/IdentityType" + }, + "role_name": { + "$ref": "#/components/schemas/ProjectRole" + } + }, + "required": [ + "identity_id", + "identity_type", + "role_name" + ] + }, + "ProjectUpdate": { + "description": "Updateable properties of a `Project`", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, + "Rack": { + "description": "View of an Rack", + "type": "object", + "properties": { + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "id", + "time_created", + "time_modified" + ] + }, + "RackResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Rack" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" ] }, - "ProjectCreate": { - "description": "Create-time parameters for a `Project`", + "Role": { + "description": "View of a Role", "type": "object", "properties": { "description": { "type": "string" }, "name": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/RoleName" } }, "required": [ @@ -15640,7 +16413,14 @@ "name" ] }, - "ProjectResultsPage": { + "RoleName": { + "title": "A name for a built-in role", + "description": "Role names consist of two string components separated by dot (\".\").", + "type": "string", + "pattern": "[a-z-]+\\.[a-z-]+", + "maxLength": 63 + }, + "RoleResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -15648,7 +16428,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/Project" + "$ref": "#/components/schemas/Role" } }, "next_page": { @@ -15661,77 +16441,284 @@ "items" ] }, - "ProjectRole": { - "type": "string", - "enum": [ - "admin", - "collaborator", - "viewer" + "Route": { + "description": "A route to a destination network through a gateway address.", + "type": "object", + "properties": { + "dst": { + "description": "The route destination.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "gw": { + "description": "The route gateway.", + "type": "string", + "format": "ip" + }, + "vid": { + "nullable": true, + "description": "VLAN id the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "dst", + "gw" ] }, - "ProjectRolePolicy": { - "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", + "RouteConfig": { + "description": "Route configuration data associated with a switch port configuration.", "type": "object", "properties": { - "role_assignments": { - "description": "Roles directly assigned on this resource", + "routes": { + "description": "The set of routes assigned to a switch port.", "type": "array", "items": { - "$ref": "#/components/schemas/ProjectRoleRoleAssignment" + "$ref": "#/components/schemas/Route" } } }, "required": [ - "role_assignments" + "routes" ] }, - "ProjectRoleRoleAssignment": { - "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", - "type": "object", - "properties": { - "identity_id": { - "type": "string", - "format": "uuid" + "RouteDestination": { + "description": "A `RouteDestination` is used to match traffic with a routing rule, on the destination of that traffic.\n\nWhen traffic is to be sent to a destination that is within a given `RouteDestination`, the corresponding `RouterRoute` applies, and traffic will be forward to the `RouteTarget` for that rule.", + "oneOf": [ + { + "description": "Route applies to traffic destined for a specific IP address", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] }, - "identity_type": { - "$ref": "#/components/schemas/IdentityType" + { + "description": "Route applies to traffic destined for a specific IP subnet", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip_net" + ] + }, + "value": { + "$ref": "#/components/schemas/IpNet" + } + }, + "required": [ + "type", + "value" + ] }, - "role_name": { - "$ref": "#/components/schemas/ProjectRole" + { + "description": "Route applies to traffic destined for the given VPC.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Route applies to traffic", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "RouteTarget": { + "description": "A `RouteTarget` describes the possible locations that traffic matching a route destination can be sent.", + "oneOf": [ + { + "description": "Forward traffic to a particular IP address.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to a VPC", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to a VPC Subnet", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to a specific instance", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "instance" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to an internet gateway", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internet_gateway" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Drop matching traffic", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "drop" + ] + } + }, + "required": [ + "type" + ] } - }, - "required": [ - "identity_id", - "identity_type", - "role_name" ] }, - "ProjectUpdate": { - "description": "Updateable properties of a `Project`", + "RouterRoute": { + "description": "A route defines a rule that governs where traffic should be sent based on its destination.", "type": "object", "properties": { "description": { - "nullable": true, + "description": "human-readable free-form text about a resource", "type": "string" }, + "destination": { + "$ref": "#/components/schemas/RouteDestination" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "kind": { + "description": "Describes the kind of router. Set at creation. `read-only`", + "allOf": [ + { + "$ref": "#/components/schemas/RouterRouteKind" + } + ] + }, "name": { - "nullable": true, + "description": "unique, mutable, user-controlled identifier for each resource", "allOf": [ { "$ref": "#/components/schemas/Name" } ] - } - } - }, - "Rack": { - "description": "View of an Rack", - "type": "object", - "properties": { - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" + }, + "target": { + "$ref": "#/components/schemas/RouteTarget" }, "time_created": { "description": "timestamp when this resource was created", @@ -15742,59 +16729,83 @@ "description": "timestamp when this resource was last modified", "type": "string", "format": "date-time" + }, + "vpc_router_id": { + "description": "The ID of the VPC Router to which the route belongs", + "type": "string", + "format": "uuid" } }, "required": [ + "description", + "destination", "id", + "kind", + "name", + "target", "time_created", - "time_modified" - ] - }, - "RackResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/Rack" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" + "time_modified", + "vpc_router_id" ] }, - "Role": { - "description": "View of a Role", + "RouterRouteCreate": { + "description": "Create-time parameters for a `RouterRoute`", "type": "object", "properties": { "description": { "type": "string" }, + "destination": { + "$ref": "#/components/schemas/RouteDestination" + }, "name": { - "$ref": "#/components/schemas/RoleName" + "$ref": "#/components/schemas/Name" + }, + "target": { + "$ref": "#/components/schemas/RouteTarget" } }, "required": [ "description", - "name" + "destination", + "name", + "target" ] }, - "RoleName": { - "title": "A name for a built-in role", - "description": "Role names consist of two string components separated by dot (\".\").", - "type": "string", - "pattern": "[a-z-]+\\.[a-z-]+", - "maxLength": 63 + "RouterRouteKind": { + "description": "The kind of a `RouterRoute`\n\nThe kind determines certain attributes such as if the route is modifiable and describes how or where the route was created.", + "oneOf": [ + { + "description": "Determines the default destination of traffic, such as whether it goes to the internet or not.\n\n`Destination: An Internet Gateway` `Modifiable: true`", + "type": "string", + "enum": [ + "default" + ] + }, + { + "description": "Automatically added for each VPC Subnet in the VPC\n\n`Destination: A VPC Subnet` `Modifiable: false`", + "type": "string", + "enum": [ + "vpc_subnet" + ] + }, + { + "description": "Automatically added when VPC peering is established\n\n`Destination: A different VPC` `Modifiable: false`", + "type": "string", + "enum": [ + "vpc_peering" + ] + }, + { + "description": "Created by a user; see `RouteTarget`\n\n`Destination: User defined` `Modifiable: true`", + "type": "string", + "enum": [ + "custom" + ] + } + ] }, - "RoleResultsPage": { + "RouterRouteResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -15802,7 +16813,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/Role" + "$ref": "#/components/schemas/RouterRoute" } }, "next_page": { @@ -15815,50 +16826,32 @@ "items" ] }, - "Route": { - "description": "A route to a destination network through a gateway address.", + "RouterRouteUpdate": { + "description": "Updateable properties of a `RouterRoute`", "type": "object", "properties": { - "dst": { - "description": "The route destination.", + "description": { + "nullable": true, + "type": "string" + }, + "destination": { + "$ref": "#/components/schemas/RouteDestination" + }, + "name": { + "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/IpNet" + "$ref": "#/components/schemas/Name" } ] }, - "gw": { - "description": "The route gateway.", - "type": "string", - "format": "ip" - }, - "vid": { - "nullable": true, - "description": "VLAN id the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "dst", - "gw" - ] - }, - "RouteConfig": { - "description": "Route configuration data associated with a switch port configuration.", - "type": "object", - "properties": { - "routes": { - "description": "The set of routes assigned to a switch port.", - "type": "array", - "items": { - "$ref": "#/components/schemas/Route" - } + "target": { + "$ref": "#/components/schemas/RouteTarget" } }, "required": [ - "routes" + "destination", + "target" ] }, "SamlIdentityProvider": { @@ -18956,10 +19949,128 @@ "items" ] }, + "VpcRouter": { + "description": "A VPC router defines a series of rules that indicate where traffic should be sent depending on its destination.", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "kind": { + "$ref": "#/components/schemas/VpcRouterKind" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + }, + "vpc_id": { + "description": "The VPC to which the router belongs.", + "type": "string", + "format": "uuid" + } + }, + "required": [ + "description", + "id", + "kind", + "name", + "time_created", + "time_modified", + "vpc_id" + ] + }, + "VpcRouterCreate": { + "description": "Create-time parameters for a `VpcRouter`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "description", + "name" + ] + }, + "VpcRouterKind": { + "type": "string", + "enum": [ + "system", + "custom" + ] + }, + "VpcRouterResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/VpcRouter" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "VpcRouterUpdate": { + "description": "Updateable properties of a `VpcRouter`", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, "VpcSubnet": { - "description": "A VPC subnet represents a logical grouping for instances that allows network traffic between them, within a IPv4 subnetwork or optionall an IPv6 subnetwork.", + "description": "A VPC subnet represents a logical grouping for instances that allows network traffic between them, within a IPv4 subnetwork or optionally an IPv6 subnetwork.", "type": "object", "properties": { + "custom_router_id": { + "nullable": true, + "description": "ID for an attached custom router.", + "type": "string", + "format": "uuid" + }, "description": { "description": "human-readable free-form text about a resource", "type": "string" From 034dd0fe8424bb8325b2623d45639649162763f9 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 16 May 2024 15:21:41 +0100 Subject: [PATCH 08/39] Iterating. --- common/src/api/internal/shared.rs | 41 +++++- illumos-utils/src/opte/port_manager.rs | 49 +++++++ nexus/src/app/background/init.rs | 20 ++- nexus/src/app/background/mod.rs | 1 + nexus/src/app/background/vpc_routes.rs | 98 +++++++++++++ openapi/sled-agent.json | 183 +++++++++++++++++++++++++ sled-agent/src/http_entrypoints.rs | 30 +++- sled-agent/src/sled_agent.rs | 10 +- 8 files changed, 427 insertions(+), 5 deletions(-) create mode 100644 nexus/src/app/background/vpc_routes.rs diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 9d9ff083e4..96f0fecd95 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -6,13 +6,13 @@ use crate::{ address::NUM_SOURCE_NAT_PORTS, - api::external::{self, BfdMode, ImportExportPolicy, IpNet, Name}, + api::external::{self, BfdMode, ImportExportPolicy, IpNet, Name, Vni}, }; use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr}, str::FromStr, @@ -590,6 +590,43 @@ impl TryFrom<&[IpNetwork]> for IpAllowList { } } +/// A VPC route resolved into a concrete target. +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +pub struct ReifiedVpcRoute { + pub dest: IpNet, + pub target: RouterTarget, +} + +/// The target for a given router entry. +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +#[serde(tag = "type", rename_all = "snake_case", content = "value")] +pub enum RouterTarget { + Drop, + InternetGateway, + Ip(IpAddr), + VpcSubnet(IpNet), +} + +/// XX +#[derive( + Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +pub struct RouterId { + pub vni: Vni, + pub subnet: Option, +} + +/// XX +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +pub struct ReifiedVpcRouteSet { + pub id: RouterId, + pub routes: HashSet, +} + #[cfg(test)] mod tests { use crate::api::{ diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index d6b4963607..48d6bb7fcb 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -16,6 +16,10 @@ use ipnetwork::IpNetwork; use omicron_common::api::external; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; +use omicron_common::api::internal::shared::ReifiedVpcRoute; +use omicron_common::api::internal::shared::ReifiedVpcRouteSet; +use omicron_common::api::internal::shared::RouterId; +use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget; use omicron_common::api::internal::shared::SourceNatConfig; use oxide_vpc::api::AddRouterEntryReq; use oxide_vpc::api::DhcpCfg; @@ -36,6 +40,8 @@ use slog::error; use slog::info; use slog::Logger; use std::collections::BTreeMap; +use std::collections::HashMap; +use std::collections::HashSet; use std::net::IpAddr; use std::net::Ipv6Addr; use std::sync::atomic::AtomicU64; @@ -60,6 +66,13 @@ struct PortManagerInner { // Map of all ports, keyed on the interface Uuid and its kind // (which includes the Uuid of the parent instance or service) ports: Mutex>, + + // XX: vs. Hashmap? + // XX: Should this be the UUID of the VPC? The rulesets are + // arguably shared v4+v6, although today we don't yet + // allow dual-stack, let alone v6. + // Map of all current resolved routes + routes: Mutex>>, } impl PortManagerInner { @@ -86,6 +99,7 @@ impl PortManager { next_port_id: AtomicU64::new(0), underlay_ip, ports: Mutex::new(BTreeMap::new()), + routes: Mutex::new(Default::default()), }); Self { inner } @@ -400,6 +414,24 @@ impl PortManager { "route" => ?route, ); + // XX: this is probably not the right initialisation here... + // XX: VPC rules should probably come from ctl plane. + let mut routes = self.inner.routes.lock().unwrap(); + routes.entry(RouterId { vni: nic.vni, subnet: None }).or_insert_with( + || { + let mut out = HashSet::new(); + out.insert(ReifiedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway, + }); + out.insert(ReifiedVpcRoute { + dest: "::/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway, + }); + out + }, + ); + info!( self.inner.log, "Created OPTE port"; @@ -408,6 +440,23 @@ impl PortManager { Ok((port, ticket)) } + pub fn vpc_routes_list(&self) -> Vec { + let routes = self.inner.routes.lock().unwrap(); + routes + .iter() + .map(|(k, v)| ReifiedVpcRouteSet { id: *k, routes: v.clone() }) + .collect() + } + + pub fn vpc_routes_ensure(&self, new_routes: Vec) { + let mut routes = self.inner.routes.lock().unwrap(); + // *routes = new_routes; + drop(routes); + + // XX: compute deltas. + // XX: push down to OPTE. + } + /// Ensure external IPs for an OPTE port are up to date. #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] pub fn external_ips_ensure( diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index d2f940018d..6210b0dd3e 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -22,6 +22,7 @@ use super::region_replacement; use super::service_firewall_rules; use super::sync_service_zone_nat::ServiceZoneNatTracker; use super::sync_switch_configuration::SwitchPortSettingsManager; +use super::vpc_routes; use crate::app::oximeter::PRODUCER_LEASE_DURATION; use crate::app::sagas::SagaRequest; use nexus_config::BackgroundTaskConfig; @@ -100,6 +101,9 @@ pub struct BackgroundTasks { /// task handle for propagation of VPC firewall rules for Omicron services /// with external network connectivity, pub task_service_firewall_propagation: common::TaskHandle, + + /// task handle for propagation of VPC router rules to all OPTE ports + pub task_vpc_route_manager: common::TaskHandle, } impl BackgroundTasks { @@ -368,6 +372,7 @@ impl BackgroundTasks { vec![], ) }; + // Background task: service firewall rule propagation let task_service_firewall_propagation = driver.register( String::from("service_firewall_rule_propagation"), @@ -377,12 +382,24 @@ impl BackgroundTasks { ), config.service_firewall_propagation.period_secs, Box::new(service_firewall_rules::ServiceRulePropagator::new( - datastore, + datastore.clone(), )), opctx.child(BTreeMap::new()), vec![], ); + let task_vpc_route_manager = { + let watcher = vpc_routes::VpcRouteManager::new(datastore); + driver.register( + "vpc_route_manager".to_string(), + "propagates updated VPC routes to all OPTE ports".into(), + config.switch_port_settings_manager.period_secs, + Box::new(watcher), + opctx.child(BTreeMap::new()), + vec![], + ) + }; + BackgroundTasks { driver, task_internal_dns_config, @@ -404,6 +421,7 @@ impl BackgroundTasks { task_region_replacement, task_instance_watcher, task_service_firewall_propagation, + task_vpc_route_manager, } } diff --git a/nexus/src/app/background/mod.rs b/nexus/src/app/background/mod.rs index 512c782b2e..db1d23f221 100644 --- a/nexus/src/app/background/mod.rs +++ b/nexus/src/app/background/mod.rs @@ -25,5 +25,6 @@ mod service_firewall_rules; mod status; mod sync_service_zone_nat; mod sync_switch_configuration; +mod vpc_routes; pub use init::BackgroundTasks; diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs new file mode 100644 index 0000000000..c5c1aaf104 --- /dev/null +++ b/nexus/src/app/background/vpc_routes.rs @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Background task for propagating VPC routes (system and custom) to sleds. + +use super::common::BackgroundTask; +use futures::future::BoxFuture; +use futures::FutureExt; +use nexus_db_model::{Sled, SledState}; +use nexus_db_queries::{context::OpContext, db::DataStore}; +use nexus_networking::sled_client_from_address; +use nexus_types::{ + deployment::SledFilter, external_api::views::SledPolicy, identity::Asset, +}; +use omicron_common::api::{external::Vni, internal::shared::RouterId}; +use serde_json::json; +use sled_agent_client::types::ReifiedVpcRoute; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; + +pub struct VpcRouteManager { + datastore: Arc, +} + +impl VpcRouteManager { + pub fn new(datastore: Arc) -> Self { + Self { datastore } + } +} + +impl BackgroundTask for VpcRouteManager { + fn activate<'a>( + &'a mut self, + opctx: &'a OpContext, + ) -> BoxFuture<'a, serde_json::Value> { + async { + let log = &opctx.log; + + // XX: copied from omicron#5566 + let sleds = match self + .datastore + .sled_list_all_batched(opctx, SledFilter::InService) + .await + { + Ok(v) => v, + Err(e) => { + let msg = format!("failed to enumerate sleds: {:#}", e); + error!(&log, "{msg}"); + return json!({"error": msg}); + } + } + .into_iter() + .filter(|sled| { + matches!(sled.state(), SledState::Active) + && matches!(sled.policy(), SledPolicy::InService { .. }) + }); + + // Map sled db records to sled-agent clients + let sled_clients: Vec<(Sled, sled_agent_client::Client)> = sleds + .map(|sled| { + let client = sled_client_from_address( + sled.id(), + sled.address(), + &log, + ); + (sled, client) + }) + .collect(); + + // XX: actually reify rules. + let mut known_rules: HashMap> = + HashMap::new(); + + for (sled, client) in sled_clients { + let Ok(a) = client.list_vpc_routes().await else { + warn!( + log, + "failed to fetch current VPC route state from sled"; + "sled" => sled.serial_number(), + ); + continue; + }; + + // XX: Who decides what we want? Do we figure out the NICs + // here? Or take the sled at their word for what subnets + // they want? + + todo!() + } + + json!({}) + } + .boxed() + } +} diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 5da2b5c797..cf55e8ef74 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -953,6 +953,63 @@ } } }, + "/vpc-routes": { + "get": { + "summary": "Get the current state of VPC routing rules.", + "operationId": "list_vpc_routes", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ReifiedVpcRouteSet", + "type": "array", + "items": { + "$ref": "#/components/schemas/ReifiedVpcRouteSet" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Update VPC routing rules.", + "operationId": "set_vpc_routes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Array_of_ReifiedVpcRouteSet", + "type": "array", + "items": { + "$ref": "#/components/schemas/ReifiedVpcRouteSet" + } + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/zones": { "get": { "summary": "List the zones that are currently managed by the sled agent.", @@ -4252,6 +4309,42 @@ "rack_subnet" ] }, + "ReifiedVpcRoute": { + "description": "A VPC route resolved into a concrete target.", + "type": "object", + "properties": { + "dest": { + "$ref": "#/components/schemas/IpNet" + }, + "target": { + "$ref": "#/components/schemas/RouterTarget" + } + }, + "required": [ + "dest", + "target" + ] + }, + "ReifiedVpcRouteSet": { + "description": "XX", + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/RouterId" + }, + "routes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ReifiedVpcRoute" + }, + "uniqueItems": true + } + }, + "required": [ + "id", + "routes" + ] + }, "RouteConfig": { "type": "object", "properties": { @@ -4281,6 +4374,96 @@ "nexthop" ] }, + "RouterId": { + "description": "XX", + "type": "object", + "properties": { + "subnet": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "vni" + ] + }, + "RouterTarget": { + "description": "The target for a given router entry.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "drop" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internet_gateway" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc_subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/IpNet" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 99c7725fe3..f41cd5e817 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -33,7 +33,7 @@ use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::{ DiskRuntimeState, SledInstanceState, UpdateArtifactId, }; -use omicron_common::api::internal::shared::SwitchPorts; +use omicron_common::api::internal::shared::{ReifiedVpcRouteSet, SwitchPorts}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_hardware::DiskVariant; @@ -86,6 +86,8 @@ pub fn api() -> SledApiDescription { api.register(host_os_write_status_delete)?; api.register(inventory)?; api.register(bootstore_status)?; + api.register(list_vpc_routes)?; + api.register(set_vpc_routes)?; Ok(()) } @@ -1018,3 +1020,29 @@ async fn bootstore_status( .into(); Ok(HttpResponseOk(status)) } + +/// Get the current state of VPC routing rules. +#[endpoint { + method = GET, + path = "/vpc-routes", +}] +async fn list_vpc_routes( + request_context: RequestContext, +) -> Result>, HttpError> { + let sa = request_context.context(); + Ok(HttpResponseOk(sa.list_vpc_routes())) +} + +/// Update VPC routing rules. +#[endpoint { + method = PUT, + path = "/vpc-routes", +}] +async fn set_vpc_routes( + request_context: RequestContext, + body: TypedBody>, +) -> Result { + let sa = request_context.context(); + sa.set_vpc_routes(body.into_inner()); + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 39a5647420..01a6e3e9d7 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -51,7 +51,7 @@ use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; use omicron_common::api::internal::shared::{ - HostPortConfig, RackNetworkConfig, + HostPortConfig, RackNetworkConfig, ReifiedVpcRouteSet, }; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, @@ -1091,6 +1091,14 @@ impl SledAgent { self.inner.bootstore.clone() } + pub fn list_vpc_routes(&self) -> Vec { + self.inner.port_manager.vpc_routes_list() + } + + pub fn set_vpc_routes(&self, routes: Vec) { + self.inner.port_manager.vpc_routes_ensure(routes) + } + /// Return the metric producer registry. pub fn metrics_registry(&self) -> &ProducerRegistry { self.inner.metrics_manager.registry() From 87d9b26ee314736f11d7410c44fcd84adf87da09 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 17 May 2024 13:14:58 +0100 Subject: [PATCH 09/39] Route resolution (but not actual installation) --- clients/sled-agent-client/src/lib.rs | 5 + illumos-utils/src/opte/port_manager.rs | 9 + nexus/db-queries/src/db/datastore/instance.rs | 52 ++++ .../src/db/datastore/network_interface.rs | 22 ++ nexus/db-queries/src/db/datastore/vpc.rs | 280 ++++++++++++++++++ nexus/src/app/background/vpc_routes.rs | 153 +++++++++- schema/crdb/vpc-subnet-routing/up02.sql | 0 7 files changed, 512 insertions(+), 9 deletions(-) create mode 100644 schema/crdb/vpc-subnet-routing/up02.sql diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index a0145af910..58a1fc0d14 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -36,6 +36,7 @@ progenitor::generate_api!( RouteConfig = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, IpNet = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] } + RouterId = { derives = [PartialEq, Eq, Hash, Debug, Deserialize, Serialize] }, }, //TODO trade the manual transformations later in this file for the // replace directives below? @@ -51,6 +52,10 @@ progenitor::generate_api!( IpNetwork = ipnetwork::IpNetwork, PortFec = omicron_common::api::internal::shared::PortFec, PortSpeed = omicron_common::api::internal::shared::PortSpeed, + RouterId = omicron_common::api::internal::shared::RouterId, + ReifiedVpcRoute = omicron_common::api::internal::shared::ReifiedVpcRoute, + ReifiedVpcRouteSet = omicron_common::api::internal::shared::ReifiedVpcRouteSet, + RouterTarget = omicron_common::api::internal::shared::RouterTarget, SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig, Vni = omicron_common::api::external::Vni, NetworkInterface = omicron_common::api::internal::shared::NetworkInterface, diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 48d6bb7fcb..02b85e37bd 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -416,6 +416,7 @@ impl PortManager { // XX: this is probably not the right initialisation here... // XX: VPC rules should probably come from ctl plane. + // XX: need to delete safely after. let mut routes = self.inner.routes.lock().unwrap(); routes.entry(RouterId { vni: nic.vni, subnet: None }).or_insert_with( || { @@ -431,6 +432,9 @@ impl PortManager { out }, ); + routes + .entry(RouterId { vni: nic.vni, subnet: Some(nic.subnet) }) + .or_insert_with(|| HashSet::default()); info!( self.inner.log, @@ -450,6 +454,11 @@ impl PortManager { pub fn vpc_routes_ensure(&self, new_routes: Vec) { let mut routes = self.inner.routes.lock().unwrap(); + error!( + self.inner.log, + "I got routes I don't know what to do with!"; + "route_set" => format!("{new_routes:?}") + ); // *routes = new_routes; drop(routes); diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index ce40e20501..cd12cb6793 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -441,6 +441,58 @@ impl DataStore { Ok(result) } + /// Lists all instances on in-service sleds with active Propolis VMM + /// processes, returning the instance along with the VMM on which it's + /// running, the sled on which the VMM is running, and the project that owns + /// the instance. + /// + /// The query performed by this function is paginated by the sled's UUID. + pub async fn instance_and_vpc_list_by_sled_agent( + &self, + opctx: &OpContext, + pagparams: &DataPageParams<'_, Uuid>, + ) -> ListResultVec<(Sled, Instance, Vmm, Project)> { + use crate::db::schema::{ + instance::dsl as instance_dsl, project::dsl as project_dsl, + sled::dsl as sled_dsl, vmm::dsl as vmm_dsl, + }; + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + let result = paginated(sled_dsl::sled, sled_dsl::id, pagparams) + .filter(sled_dsl::time_deleted.is_null()) + .sled_filter(SledFilter::InService) + .inner_join( + vmm_dsl::vmm + .on(vmm_dsl::sled_id + .eq(sled_dsl::id) + .and(vmm_dsl::time_deleted.is_null())) + .inner_join( + instance_dsl::instance + .on(instance_dsl::id + .eq(vmm_dsl::instance_id) + .and(instance_dsl::time_deleted.is_null())) + .inner_join( + project_dsl::project.on(project_dsl::id + .eq(instance_dsl::project_id) + .and(project_dsl::time_deleted.is_null())), + ), + ), + ) + .sled_filter(SledFilter::InService) + .select(( + Sled::as_select(), + Instance::as_select(), + Vmm::as_select(), + Project::as_select(), + )) + .load_async::<(Sled, Instance, Vmm, Project)>(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(result) + } + pub async fn project_delete_instance( &self, opctx: &OpContext, diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 733e4ef32b..b426c8b472 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -609,6 +609,28 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + /// Retrieve the primary network interface for a given instance. + pub async fn instance_get_primary_network_interface( + &self, + opctx: &OpContext, + authz_instance: &authz::Instance, + ) -> LookupResult { + opctx.authorize(authz::Action::ListChildren, authz_instance).await?; + + use db::schema::instance_network_interface::dsl; + dsl::instance_network_interface + .filter(dsl::time_deleted.is_null()) + .filter(dsl::instance_id.eq(authz_instance.id())) + .filter(dsl::is_primary.eq(true)) + .select(InstanceNetworkInterface::as_select()) + .limit(1) + .first_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + /// Get network interface associated with a given probe. pub async fn probe_get_network_interface( &self, diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index d200f67663..a6a69e388b 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -5,6 +5,7 @@ //! [`DataStore`] methods on [`Vpc`]s. use super::DataStore; +use super::SQL_BATCH_SIZE; use crate::authz; use crate::context::OpContext; use crate::db; @@ -35,6 +36,7 @@ use crate::db::model::VpcSubnetUpdate; use crate::db::model::VpcUpdate; use crate::db::model::{Ipv4Net, Ipv6Net}; use crate::db::pagination::paginated; +use crate::db::pagination::Paginator; use crate::db::queries::vpc::InsertVpcQuery; use crate::db::queries::vpc::VniSearchIter; use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery; @@ -54,6 +56,7 @@ use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::InternalContext; +use omicron_common::api::external::IpNet; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::LookupType; @@ -63,9 +66,13 @@ use omicron_common::api::external::RouteTarget; use omicron_common::api::external::RouterRouteKind as ExternalRouteKind; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::Vni as ExternalVni; +use omicron_common::api::internal::shared::ReifiedVpcRoute; +use omicron_common::api::internal::shared::RouterTarget; use ref_cast::RefCast; use std::collections::BTreeMap; +use std::collections::HashMap; use std::collections::HashSet; +use std::net::IpAddr; use uuid::Uuid; impl DataStore { @@ -1403,6 +1410,279 @@ impl DataStore { }}).await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + + /// Look up a VPC by VNI. + pub async fn vpc_get_system_router( + &self, + opctx: &OpContext, + vpc_id: Uuid, + ) -> LookupResult { + // use db::schema::vpc::dsl as vpc_dsl; + use db::schema::vpc::dsl as vpc_dsl; + use db::schema::vpc_router::dsl as router_dsl; + + vpc_dsl::vpc + .inner_join( + router_dsl::vpc_router + .on(router_dsl::id.eq(vpc_dsl::system_router_id)), + ) + .filter(vpc_dsl::time_deleted.is_null()) + .filter(vpc_dsl::id.eq(vpc_id)) + .filter(router_dsl::time_deleted.is_null()) + .filter(router_dsl::vpc_id.eq(vpc_id)) + .select(VpcRouter::as_select()) + .limit(1) + .first_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Vpc, + LookupType::ById(vpc_id), + ), + ) + }) + } + + /// Fetch all active custom routers (and their parent subnets) + /// in a VPC. + pub async fn vpc_get_active_custom_routers( + &self, + opctx: &OpContext, + vpc_id: Uuid, + ) -> ListResultVec<(VpcSubnet, VpcRouter)> { + use db::schema::vpc_router::dsl as router_dsl; + use db::schema::vpc_subnet::dsl as subnet_dsl; + + subnet_dsl::vpc_subnet + .inner_join( + router_dsl::vpc_router.on(router_dsl::id + .nullable() + .eq(subnet_dsl::custom_router_id)), + ) + .filter(subnet_dsl::time_deleted.is_null()) + .filter(subnet_dsl::vpc_id.is_null()) + .filter(router_dsl::time_deleted.is_null()) + .select((VpcSubnet::as_select(), VpcRouter::as_select())) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Vpc, + LookupType::ById(vpc_id), + ), + ) + }) + } + + /// Resolve all targets in a router into concrete details. + pub async fn vpc_resolve_router_rules( + &self, + opctx: &OpContext, + vpc_router_id: Uuid, + ) -> Result, Error> { + // Get all rules in target router. + opctx.check_complex_operations_allowed()?; + + let (.., authz_project, authz_vpc, authz_router) = + db::lookup::LookupPath::new(opctx, self) + .vpc_router_id(vpc_router_id) + .lookup_for(authz::Action::ListChildren) + .await + .internal_context("lookup built-in services project")?; + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + let mut all_rules = vec![]; + while let Some(p) = paginator.next() { + let batch = self + .vpc_router_route_list( + opctx, + &authz_router, + &PaginatedBy::Id(p.current_pagparams()), + ) + .await?; + paginator = p + .found_batch(&batch, &|s: &nexus_db_model::RouterRoute| s.id()); + all_rules.extend(batch); + } + + // XXX: transaction based on generation number? + let mut subnet_names = HashSet::new(); + let mut vpc_names = HashSet::new(); + let mut inetgw_names = HashSet::new(); + let mut instance_names = HashSet::new(); + for rule in &all_rules { + match &rule.target.0 { + RouteTarget::Vpc(n) => { + vpc_names.insert(n.clone()); + } + RouteTarget::Subnet(n) => { + subnet_names.insert(n.clone()); + } + RouteTarget::Instance(n) => { + instance_names.insert(n.clone()); + } + RouteTarget::InternetGateway(n) => { + inetgw_names.insert(n.clone()); + } + _ => {} + } + + match &rule.destination.0 { + RouteDestination::Vpc(n) => { + vpc_names.insert(n.clone()); + } + RouteDestination::Subnet(n) => { + subnet_names.insert(n.clone()); + } + _ => {} + } + } + + // TODO: transact these, and/or solve in fewer queries. + let mut subnets = HashMap::new(); + for name in subnet_names.drain() { + if let Ok((.., subnet)) = db::lookup::LookupPath::new(opctx, self) + .vpc_id(authz_vpc.id()) + .vpc_subnet_name(Name::ref_cast(&name)) + .fetch() + .await + { + subnets.insert(name, subnet); + } + } + let mut vpcs = HashMap::new(); + for name in vpc_names.drain() { + if let Ok((.., vpc)) = db::lookup::LookupPath::new(opctx, self) + .project_id(authz_project.id()) + .vpc_name(Name::ref_cast(&name)) + .fetch() + .await + { + vpcs.insert(name, vpc); + } + } + let mut instances = HashMap::new(); + for name in instance_names.drain() { + if let Ok((.., authz_instance, instance)) = + db::lookup::LookupPath::new(opctx, self) + .project_id(authz_project.id()) + .instance_name(Name::ref_cast(&name)) + .fetch() + .await + { + // XXX: currently an instance can have one primary, + // and it is not dual-stack (v4 + v6). We need + // to clarify what should be resolved in the v6 case. + if let Ok(primary_nic) = self + .instance_get_primary_network_interface( + opctx, + &authz_instance, + ) + .await + { + instances.insert(name, (instance, primary_nic)); + } + } + } + // TODO: validate names of Internet Gateways. + + // See the discussion in `resolve_firewall_rules_for_sled_agent` on + // how we should resolve name misses in route resolution. + // This method adopts the same strategy: a lookup failure corresponds + // to a NO-OP rule. + let mut out = HashSet::new(); + for rule in all_rules { + // Some dests/targets (e.g., subnet) resolve to *several* specifiers + // to handle both v4 and v6. The user-facing API will prevent severe + // mistakes on naked IPs/CIDRs (mixed v4/6), but we need to be smarter + // around named entities here. + let (v4_dest, v6_dest) = match rule.destination.0 { + RouteDestination::Ip(ip @ IpAddr::V4(_)) => { + (Some(IpNet::single(ip)), None) + } + RouteDestination::Ip(ip @ IpAddr::V6(_)) => { + (None, Some(IpNet::single(ip))) + } + RouteDestination::IpNet(ip @ IpNet::V4(_)) => (Some(ip), None), + RouteDestination::IpNet(ip @ IpNet::V6(_)) => (None, Some(ip)), + RouteDestination::Subnet(n) => subnets + .get(&n) + .map(|s| { + ( + Some(s.ipv4_block.0.into()), + Some(s.ipv6_block.0.into()), + ) + }) + .unwrap_or_default(), + + // TODO: VPC peering. + RouteDestination::Vpc(_) => (None, None), + }; + + let (v4_target, v6_target) = match rule.target.0 { + RouteTarget::Ip(ip @ IpAddr::V4(_)) => { + (Some(RouterTarget::Ip(ip)), None) + } + RouteTarget::Ip(ip @ IpAddr::V6(_)) => { + (None, Some(RouterTarget::Ip(ip))) + } + RouteTarget::Subnet(n) => subnets + .get(&n) + .map(|s| { + ( + Some(RouterTarget::VpcSubnet( + s.ipv4_block.0.into(), + )), + Some(RouterTarget::VpcSubnet( + s.ipv6_block.0.into(), + )), + ) + }) + .unwrap_or_default(), + RouteTarget::Instance(n) => instances + .get(&n) + .map(|i| match i.1.ip { + // TODO: update for dual-stack v4/6. + ip @ IpNetwork::V4(_) => { + (Some(RouterTarget::Ip(ip.ip())), None) + } + ip @ IpNetwork::V6(_) => { + (None, Some(RouterTarget::Ip(ip.ip()))) + } + }) + .unwrap_or_default(), + RouteTarget::Drop => { + (Some(RouterTarget::Drop), Some(RouterTarget::Drop)) + } + + // TODO: Internet Gateways. + // The semantic here is 'name match => allow', + // as the other aspect they will control is SNAT + // IP allocation. Today, presence of this rule + // allows upstream regardless of name. + RouteTarget::InternetGateway(_n) => ( + Some(RouterTarget::InternetGateway), + Some(RouterTarget::InternetGateway), + ), + + // TODO: VPC Peering. + RouteTarget::Vpc(_) => (None, None), + }; + + if let (Some(dest), Some(target)) = (v4_dest, v4_target) { + out.insert(ReifiedVpcRoute { dest, target }); + } + + if let (Some(dest), Some(target)) = (v6_dest, v6_target) { + out.insert(ReifiedVpcRoute { dest, target }); + } + } + + Ok(out) + } } #[cfg(test)] diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index c5c1aaf104..6f21d61204 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -7,19 +7,21 @@ use super::common::BackgroundTask; use futures::future::BoxFuture; use futures::FutureExt; -use nexus_db_model::{Sled, SledState}; +use nexus_db_model::{Sled, SledState, Vni}; use nexus_db_queries::{context::OpContext, db::DataStore}; use nexus_networking::sled_client_from_address; use nexus_types::{ deployment::SledFilter, external_api::views::SledPolicy, identity::Asset, + identity::Resource, }; -use omicron_common::api::{external::Vni, internal::shared::RouterId}; +use omicron_common::api::internal::shared::ReifiedVpcRoute; +use omicron_common::api::internal::shared::{ReifiedVpcRouteSet, RouterId}; use serde_json::json; -use sled_agent_client::types::ReifiedVpcRoute; use std::{ collections::{HashMap, HashSet}, sync::Arc, }; +use uuid::Uuid; pub struct VpcRouteManager { datastore: Arc, @@ -38,6 +40,7 @@ impl BackgroundTask for VpcRouteManager { ) -> BoxFuture<'a, serde_json::Value> { async { let log = &opctx.log; + // let mut paginator = Paginator::new(MAX_SLED_AGENTS); // XX: copied from omicron#5566 let sleds = match self @@ -71,11 +74,12 @@ impl BackgroundTask for VpcRouteManager { .collect(); // XX: actually reify rules. - let mut known_rules: HashMap> = + let mut known_rules: HashMap> = HashMap::new(); + let mut db_routers = HashMap::new(); for (sled, client) in sled_clients { - let Ok(a) = client.list_vpc_routes().await else { + let Ok(route_sets) = client.list_vpc_routes().await else { warn!( log, "failed to fetch current VPC route state from sled"; @@ -84,11 +88,142 @@ impl BackgroundTask for VpcRouteManager { continue; }; - // XX: Who decides what we want? Do we figure out the NICs - // here? Or take the sled at their word for what subnets - // they want? + let route_sets = route_sets.into_inner(); - todo!() + // Lookup all missing (VNI, subnet) pairs we need from this sled. + for set in &route_sets { + let system_route = + RouterId { vni: set.id.vni, subnet: None }; + + if db_routers.contains_key(&system_route) { + continue; + } + + let db_vni = Vni(set.id.vni); + let Ok(vpc) = + self.datastore.resolve_vni_to_vpc(opctx, db_vni).await + else { + error!( + log, + "failed to fetch VPC from VNI"; + "sled" => sled.serial_number(), + "vni" => format!("{db_vni:?}") + ); + continue; + }; + + let vpc_id = vpc.identity().id; + + let Ok(system_router) = self + .datastore + .vpc_get_system_router(opctx, vpc_id) + .await + else { + error!( + log, + "failed to fetch system router for VPC"; + "vpc" => vpc_id.to_string() + ); + continue; + }; + + let Ok(custom_routers) = self + .datastore + .vpc_get_active_custom_routers(opctx, vpc_id) + .await + else { + error!( + log, + "failed to fetch custom routers for VPC"; + "vpc" => vpc_id.to_string() + ); + continue; + }; + + db_routers.insert(system_route, system_router); + db_routers.extend(custom_routers.into_iter().map( + |(subnet, router)| { + ( + RouterId { + vni: set.id.vni, + subnet: Some(subnet.ipv4_block.0.into()), + }, + router, + ) + }, + )); + // XX: do this right / unify v4 and v6 + // db_routers.extend(custom_routers.into_iter().map( + // |(subnet, router)| { + // ( + // RouterId { + // vni: set.id.vni, + // subnet: Some(subnet.ipv6_block.0.into()), + // }, + // router, + // ) + // }, + // )); + } + + let mut to_push = HashMap::new(); + + // reify into known_rules on an as-needed basis. + for set in &route_sets { + let Some(db_router) = db_routers.get(&set.id) else { + // The sled wants to know about rules for a VPC + // subnet with no custom router set. Send them + // the empty list. + to_push.insert(set.id, HashSet::new()); + continue; + }; + + let router_id = db_router.id(); + + // We may have already resolved the rules for this + // router in a previous call. + if let Some(rules) = known_rules.get(&router_id) { + to_push.insert(set.id, rules.clone()); + continue; + } + + match self + .datastore + .vpc_resolve_router_rules( + opctx, + db_router.identity().id, + ) + .await + { + Ok(rules) => { + to_push.insert(set.id, rules.clone()); + known_rules.insert(router_id, rules); + } + Err(e) => { + error!( + &log, + "failed to compute subnet routes"; + "router" => router_id.to_string(), + "err" => e.to_string() + ); + } + } + } + + let to_push = to_push + .into_iter() + .map(|(id, routes)| ReifiedVpcRouteSet { id, routes }) + .collect(); + + if let Err(e) = client.set_vpc_routes(&to_push).await { + error!( + log, + "failed to push new VPC route state from sled"; + "sled" => sled.serial_number(), + "err" => format!("{e}") + ); + continue; + }; } json!({}) diff --git a/schema/crdb/vpc-subnet-routing/up02.sql b/schema/crdb/vpc-subnet-routing/up02.sql new file mode 100644 index 0000000000..e69de29bb2 From 568f44c426284a3489523079fa856684a91119ae Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 17 May 2024 13:47:32 +0100 Subject: [PATCH 10/39] Wrong ID in router initialisation. --- nexus/db-queries/src/db/datastore/vpc.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index a6a69e388b..5cbf0a5edb 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -254,6 +254,7 @@ impl DataStore { opctx: &OpContext, authz_router: &authz::VpcRouter, ) -> Result<(), Error> { + use crate::db::fixed_data::vpc::SERVICES_VPC; use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET; use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET_ROUTE_ID; use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; @@ -293,7 +294,7 @@ impl DataStore { let route = RouterRoute::for_subnet( route_id, - *SERVICES_VPC_ID, + SERVICES_VPC.system_router_id, vpc_subnet.name().clone().into(), ) .expect("builtin service names are short enough for route naming"); From 2a25d74fd88feb42deb05fbbfebb0f7d002ec897 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 17 May 2024 21:10:52 +0100 Subject: [PATCH 11/39] Rule insert/delete with OPTE. --- illumos-utils/src/opte/firewall_rules.rs | 19 +----- illumos-utils/src/opte/mod.rs | 31 +++++++++ illumos-utils/src/opte/port.rs | 9 +++ illumos-utils/src/opte/port_manager.rs | 80 +++++++++++++++++++++--- sled-agent/src/http_entrypoints.rs | 2 +- sled-agent/src/sled_agent.rs | 7 ++- 6 files changed, 119 insertions(+), 29 deletions(-) diff --git a/illumos-utils/src/opte/firewall_rules.rs b/illumos-utils/src/opte/firewall_rules.rs index 02882a226b..6df07b3a54 100644 --- a/illumos-utils/src/opte/firewall_rules.rs +++ b/illumos-utils/src/opte/firewall_rules.rs @@ -4,6 +4,7 @@ //! Convert Omicron VPC firewall rules to OPTE firewall rules. +use super::net_to_cidr; use crate::opte::params::VpcFirewallRule; use crate::opte::Vni; use macaddr::MacAddr6; @@ -19,11 +20,6 @@ use oxide_vpc::api::Filters; use oxide_vpc::api::FirewallAction; use oxide_vpc::api::FirewallRule; use oxide_vpc::api::IpAddr; -use oxide_vpc::api::IpCidr; -use oxide_vpc::api::Ipv4Cidr; -use oxide_vpc::api::Ipv4PrefixLen; -use oxide_vpc::api::Ipv6Cidr; -use oxide_vpc::api::Ipv6PrefixLen; use oxide_vpc::api::Ports; use oxide_vpc::api::ProtoFilter; use oxide_vpc::api::Protocol; @@ -70,23 +66,12 @@ impl FromVpcFirewallRule for VpcFirewallRule { { Address::Ip(IpAddr::Ip4(net.ip().into())) } - HostIdentifier::Ip(IpNet::V4(net)) => { - Address::Subnet(IpCidr::Ip4(Ipv4Cidr::new( - net.ip().into(), - Ipv4PrefixLen::new(net.prefix()).unwrap(), - ))) - } HostIdentifier::Ip(IpNet::V6(net)) if net.prefix() == 128 => { Address::Ip(IpAddr::Ip6(net.ip().into())) } - HostIdentifier::Ip(IpNet::V6(net)) => { - Address::Subnet(IpCidr::Ip6(Ipv6Cidr::new( - net.ip().into(), - Ipv6PrefixLen::new(net.prefix()).unwrap(), - ))) - } + HostIdentifier::Ip(ip) => Address::Subnet(net_to_cidr(*ip)), HostIdentifier::Vpc(vni) => { Address::Vni(Vni::new(u32::from(*vni)).unwrap()) } diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index d06b6b26e5..367a1836c4 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -18,6 +18,14 @@ mod port; mod port_manager; pub use firewall_rules::opte_firewall_rules; +use omicron_common::api::external::IpNet; +use omicron_common::api::internal::shared; +use oxide_vpc::api::IpCidr; +use oxide_vpc::api::Ipv4Cidr; +use oxide_vpc::api::Ipv4PrefixLen; +use oxide_vpc::api::Ipv6Cidr; +use oxide_vpc::api::Ipv6PrefixLen; +use oxide_vpc::api::RouterTarget; pub use port::Port; pub use port_manager::PortManager; pub use port_manager::PortTicket; @@ -63,3 +71,26 @@ impl Gateway { &self.ip } } + +fn net_to_cidr(net: IpNet) -> IpCidr { + match net { + IpNet::V4(net) => IpCidr::Ip4(Ipv4Cidr::new( + net.ip().into(), + Ipv4PrefixLen::new(net.prefix()).unwrap(), + )), + IpNet::V6(net) => IpCidr::Ip6(Ipv6Cidr::new( + net.ip().into(), + Ipv6PrefixLen::new(net.prefix()).unwrap(), + )), + } +} + +fn router_target_opte(target: &shared::RouterTarget) -> RouterTarget { + use shared::RouterTarget::*; + match target { + Drop => RouterTarget::Drop, + InternetGateway => RouterTarget::InternetGateway, + Ip(ip) => RouterTarget::Ip((*ip).into()), + VpcSubnet(net) => RouterTarget::VpcSubnet(net_to_cidr(*net)), + } +} diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 6fbb89c450..013cc5bc61 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -7,6 +7,7 @@ use crate::opte::Gateway; use crate::opte::Vni; use macaddr::MacAddr6; +use omicron_common::api::external::IpNet; use std::net::IpAddr; use std::sync::Arc; @@ -22,6 +23,8 @@ struct PortInner { slot: u8, // Geneve VNI for the VPC vni: Vni, + // Subnet the port belong to within the VPC. + subnet: IpNet, // Information about the virtual gateway, aka OPTE gateway: Gateway, // TODO-remove(#2932): Remove this once we can put Viona directly on top of an @@ -89,6 +92,7 @@ impl Port { mac: MacAddr6, slot: u8, vni: Vni, + subnet: IpNet, gateway: Gateway, vnic: String, ) -> Self { @@ -99,6 +103,7 @@ impl Port { mac, slot, vni, + subnet, gateway, vnic, }), @@ -126,6 +131,10 @@ impl Port { &self.inner.vni } + pub fn subnet(&self) -> &IpNet { + &self.inner.subnet + } + pub fn vnic_name(&self) -> &str { &self.inner.vnic } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 02b85e37bd..dc6e96a5b8 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -22,6 +22,7 @@ use omicron_common::api::internal::shared::RouterId; use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget; use omicron_common::api::internal::shared::SourceNatConfig; use oxide_vpc::api::AddRouterEntryReq; +use oxide_vpc::api::DelRouterEntryReq; use oxide_vpc::api::DhcpCfg; use oxide_vpc::api::ExternalIpCfg; use oxide_vpc::api::IpCfg; @@ -341,6 +342,7 @@ impl PortManager { mac, nic.slot, vni, + nic.subnet, gateway, vnic, ); @@ -452,18 +454,78 @@ impl PortManager { .collect() } - pub fn vpc_routes_ensure(&self, new_routes: Vec) { + pub fn vpc_routes_ensure( + &self, + new_routes: Vec, + ) -> Result<(), Error> { let mut routes = self.inner.routes.lock().unwrap(); - error!( - self.inner.log, - "I got routes I don't know what to do with!"; - "route_set" => format!("{new_routes:?}") - ); - // *routes = new_routes; + let mut deltas = HashMap::new(); + for set in new_routes { + let old = routes.get(&set.id); + + let (to_add, to_delete) = if let Some(old) = old { + ( + set.routes.difference(old).cloned().collect(), + old.difference(&set.routes).cloned().collect(), + ) + } else { + (set.routes.clone(), HashSet::new()) + }; + deltas.insert(set.id, (to_add, to_delete)); + + routes.insert(set.id, set.routes); + } drop(routes); - // XX: compute deltas. - // XX: push down to OPTE. + let ports = self.inner.ports.lock().unwrap(); + #[cfg(target_os = "illumos")] + let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?; + + for port in ports.values() { + let vni = external::Vni::try_from(port.vni().as_u32()).unwrap(); + let system_id = RouterId { vni, subnet: None }; + let system_delta = deltas.get(&system_id); + + let custom_id = RouterId { vni, subnet: Some(*port.subnet()) }; + + let custom_delta = deltas.get(&custom_id); + + #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] + for (class, delta) in [ + (RouterClass::System, system_delta), + (RouterClass::Custom, custom_delta), + ] { + let Some((to_add, to_delete)) = delta else { + continue; + }; + + for route in to_delete { + let route = DelRouterEntryReq { + class, + port_name: port.name().into(), + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + }; + + #[cfg(target_os = "illumos")] + hdl.del_router_entry(&route)?; + } + + for route in to_add { + let route = AddRouterEntryReq { + class, + port_name: port.name().into(), + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + }; + + #[cfg(target_os = "illumos")] + hdl.add_router_entry(&route)?; + } + } + } + + Ok(()) } /// Ensure external IPs for an OPTE port are up to date. diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index f41cd5e817..bf85bb49a7 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -1043,6 +1043,6 @@ async fn set_vpc_routes( body: TypedBody>, ) -> Result { let sa = request_context.context(); - sa.set_vpc_routes(body.into_inner()); + sa.set_vpc_routes(body.into_inner())?; Ok(HttpResponseUpdatedNoContent()) } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 01a6e3e9d7..263db9555e 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -1095,8 +1095,11 @@ impl SledAgent { self.inner.port_manager.vpc_routes_list() } - pub fn set_vpc_routes(&self, routes: Vec) { - self.inner.port_manager.vpc_routes_ensure(routes) + pub fn set_vpc_routes( + &self, + routes: Vec, + ) -> Result<(), Error> { + self.inner.port_manager.vpc_routes_ensure(routes).map_err(Error::from) } /// Return the metric producer registry. From c718c8dc0e6634cec6eea70f6db6b79729ada93d Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 17 May 2024 23:23:21 +0100 Subject: [PATCH 12/39] Correctly instantiate router rules if existing --- illumos-utils/src/opte/port_manager.rs | 138 +++++++++++-------------- sled-agent/src/instance.rs | 1 + sled-agent/src/probe_manager.rs | 1 + sled-agent/src/services.rs | 10 +- 4 files changed, 72 insertions(+), 78 deletions(-) diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index dc6e96a5b8..8e95c4e46b 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -31,7 +31,6 @@ use oxide_vpc::api::Ipv4Cfg; use oxide_vpc::api::Ipv6Cfg; use oxide_vpc::api::MacAddr; use oxide_vpc::api::RouterClass; -use oxide_vpc::api::RouterTarget; use oxide_vpc::api::SNat4Cfg; use oxide_vpc::api::SNat6Cfg; use oxide_vpc::api::SetExternalIpsReq; @@ -120,6 +119,7 @@ impl PortManager { floating_ips: &[IpAddr], firewall_rules: &[VpcFirewallRule], dhcp_config: DhcpCfg, + is_service: bool, ) -> Result<(Port, PortTicket), Error> { let mac = *nic.mac; let vni = Vni::new(nic.vni).unwrap(); @@ -356,88 +356,58 @@ impl PortManager { (port, ticket) }; - // TODO: These should not be filled in like this, and should be informed - // by either our existing knowledge of current knowledge of system + custom - // routers OR we just await the router RPW filling this in for us. - // In future, ∃ VPCs *without* an Internet Gateway so we can't just - // plumb that in as well... - - // Add a router entry for this interface's subnet, directing traffic to the - // VPC subnet. - let route = AddRouterEntryReq { - class: RouterClass::System, - port_name: port_name.clone(), - dest: vpc_subnet, - target: RouterTarget::VpcSubnet(vpc_subnet), - }; - #[cfg(target_os = "illumos")] - hdl.add_router_entry(&route)?; - debug!( - self.inner.log, - "Added VPC Subnet router entry"; - "port_name" => &port_name, - "route" => ?route, - ); - - // TODO-remove - // - // See https://github.com/oxidecomputer/omicron/issues/1336 - // - // This is another part of the workaround, allowing reply traffic from - // the guest back out. Normally, OPTE would drop such traffic at the - // router layer, as it has no route for that external IP address. This - // allows such traffic through. - // - // Note that this exact rule will eventually be included, since it's one - // of the default routing rules in the VPC System Router. However, that - // will likely be communicated in a different way, or could be modified, - // and this specific call should be removed in favor of sending the - // routing rules the control plane provides. - // - // This rule sends all traffic that has no better match to the gateway. - let dest = match vpc_subnet { - IpCidr::Ip4(_) => "0.0.0.0/0", - IpCidr::Ip6(_) => "::/0", - } - .parse() - .unwrap(); - let route = AddRouterEntryReq { - class: RouterClass::System, - port_name: port_name.clone(), - dest, - target: RouterTarget::InternetGateway, - }; - #[cfg(target_os = "illumos")] - hdl.add_router_entry(&route)?; - debug!( - self.inner.log, - "Added default internet gateway route entry"; - "port_name" => &port_name, - "route" => ?route, - ); - - // XX: this is probably not the right initialisation here... - // XX: VPC rules should probably come from ctl plane. - // XX: need to delete safely after. + // XX: need to delete safely after all subnet-holders leave + // to not get flooded with useless rules. let mut routes = self.inner.routes.lock().unwrap(); - routes.entry(RouterId { vni: nic.vni, subnet: None }).or_insert_with( - || { + let system_routes = routes + .entry(RouterId { vni: nic.vni, subnet: None }) + .or_insert_with(|| { let mut out = HashSet::new(); - out.insert(ReifiedVpcRoute { - dest: "0.0.0.0/0".parse().unwrap(), - target: ApiRouterTarget::InternetGateway, - }); - out.insert(ReifiedVpcRoute { - dest: "::/0".parse().unwrap(), - target: ApiRouterTarget::InternetGateway, - }); + // Services do not talk to one another via OPTE, but do need + // to reach out over the Internet *before* nexus is up to give + // us real rules. The easiest bet is to instantiate these here. + if is_service { + out.insert(ReifiedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway, + }); + out.insert(ReifiedVpcRoute { + dest: "::/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway, + }); + } out - }, - ); - routes + }) + .clone(); + + let custom_routes = routes .entry(RouterId { vni: nic.vni, subnet: Some(nic.subnet) }) .or_insert_with(|| HashSet::default()); + for (class, routes) in [ + (RouterClass::System, &system_routes), + (RouterClass::Custom, custom_routes), + ] { + for route in routes { + let route = AddRouterEntryReq { + class, + port_name: port_name.clone(), + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + }; + + #[cfg(target_os = "illumos")] + hdl.add_router_entry(&route)?; + + debug!( + self.inner.log, + "Added router entry"; + "port_name" => &port_name, + "route" => ?route, + ); + } + } + info!( self.inner.log, "Created OPTE port"; @@ -509,6 +479,13 @@ impl PortManager { #[cfg(target_os = "illumos")] hdl.del_router_entry(&route)?; + + debug!( + self.inner.log, + "Removed router entry"; + "port_name" => &port.name(), + "route" => ?route, + ); } for route in to_add { @@ -521,6 +498,13 @@ impl PortManager { #[cfg(target_os = "illumos")] hdl.add_router_entry(&route)?; + + debug!( + self.inner.log, + "Added router entry"; + "port_name" => &port.name(), + "route" => ?route, + ); } } } diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 271eceb556..e43e13f96f 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -1330,6 +1330,7 @@ impl InstanceRunner { floating_ips, &self.firewall_rules, self.dhcp_config.clone(), + false, )?; opte_ports.push(port); } diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index 16559039a2..67357c9512 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -239,6 +239,7 @@ impl ProbeManagerInner { priority: VpcFirewallRulePriority(100), }], DhcpCfg::default(), + false, )?; let installed_zone = ZoneBuilderFactory::default() diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index c9a5014402..b8bde2bfc8 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1266,7 +1266,15 @@ impl ServiceManager { // config allows outbound access which is enough for // Boundary NTP which needs to come up before Nexus. let port = port_manager - .create_port(nic, snat, None, floating_ips, &[], DhcpCfg::default()) + .create_port( + nic, + snat, + None, + floating_ips, + &[], + DhcpCfg::default(), + true, + ) .map_err(|err| Error::ServicePortCreation { service: zone_type_str.clone(), err: Box::new(err), From 06eaaabb60f25818ba78cdad5981dcab1b0c2a02 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Mon, 20 May 2024 11:58:13 +0100 Subject: [PATCH 13/39] Feed Clippy. --- illumos-utils/src/opte/mod.rs | 1 + illumos-utils/src/opte/port.rs | 68 ++++++++++++-------------- illumos-utils/src/opte/port_manager.rs | 43 +++++++++++----- sled-agent/src/instance.rs | 14 +++--- sled-agent/src/probe_manager.rs | 20 ++++---- sled-agent/src/services.rs | 16 +++--- 6 files changed, 86 insertions(+), 76 deletions(-) diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 367a1836c4..f6ef186808 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -27,6 +27,7 @@ use oxide_vpc::api::Ipv6Cidr; use oxide_vpc::api::Ipv6PrefixLen; use oxide_vpc::api::RouterTarget; pub use port::Port; +pub use port_manager::PortCreateParams; pub use port_manager::PortManager; pub use port_manager::PortTicket; diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 013cc5bc61..38ba1b8c1c 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -12,21 +12,22 @@ use std::net::IpAddr; use std::sync::Arc; #[derive(Debug)] -struct PortInner { - // Name of the port as identified by OPTE - name: String, - // IP address within the VPC Subnet - ip: IpAddr, - // VPC-private MAC address - mac: MacAddr6, - // Emulated PCI slot for the guest NIC, passed to Propolis - slot: u8, - // Geneve VNI for the VPC - vni: Vni, - // Subnet the port belong to within the VPC. - subnet: IpNet, - // Information about the virtual gateway, aka OPTE - gateway: Gateway, +pub struct PortData { + /// Name of the port as identified by OPTE + pub(crate) name: String, + /// IP address within the VPC Subnet + pub(crate) ip: IpAddr, + /// VPC-private MAC address + pub(crate) mac: MacAddr6, + /// Emulated PCI slot for the guest NIC, passed to Propolis + pub(crate) slot: u8, + /// Geneve VNI for the VPC + pub(crate) vni: Vni, + /// Subnet the port belong to within the VPC. + pub(crate) subnet: IpNet, + /// Information about the virtual gateway, aka OPTE + pub(crate) gateway: Gateway, + /// Name of the VNIC the OPTE port is bound to. // TODO-remove(#2932): Remove this once we can put Viona directly on top of an // OPTE port device. // @@ -36,7 +37,18 @@ struct PortInner { // https://github.com/oxidecomputer/opte/issues/178 for more details. This // can be changed back to a real VNIC when that is resolved, and the Drop // impl below can simplify to just call `drop(self.vnic)`. - vnic: String, + pub(crate) vnic: String, +} + +#[derive(Debug)] +struct PortInner(PortData); + +impl core::ops::Deref for PortInner { + type Target = PortData; + + fn deref(&self) -> &Self::Target { + &self.0 + } } #[cfg(target_os = "illumos")] @@ -86,28 +98,8 @@ pub struct Port { } impl Port { - pub fn new( - name: String, - ip: IpAddr, - mac: MacAddr6, - slot: u8, - vni: Vni, - subnet: IpNet, - gateway: Gateway, - vnic: String, - ) -> Self { - Self { - inner: Arc::new(PortInner { - name, - ip, - mac, - slot, - vni, - subnet, - gateway, - vnic, - }), - } + pub fn new(data: PortData) -> Self { + Self { inner: Arc::new(PortInner(data)) } } pub fn ip(&self) -> &IpAddr { diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 8e95c4e46b..968854381e 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -8,6 +8,7 @@ use crate::opte::opte_firewall_rules; use crate::opte::params::DeleteVirtualNetworkInterfaceHost; use crate::opte::params::SetVirtualNetworkInterfaceHost; use crate::opte::params::VpcFirewallRule; +use crate::opte::port::PortData; use crate::opte::Error; use crate::opte::Gateway; use crate::opte::Port; @@ -85,6 +86,18 @@ impl PortManagerInner { } } +#[derive(Debug)] +/// Parameters needed to create and configure an OPTE port. +pub struct PortCreateParams<'a> { + pub nic: &'a NetworkInterface, + pub source_nat: Option, + pub ephemeral_ip: Option, + pub floating_ips: &'a [IpAddr], + pub firewall_rules: &'a [VpcFirewallRule], + pub dhcp_config: DhcpCfg, + pub is_service: bool, +} + /// The port manager controls all OPTE ports on a single host. #[derive(Debug, Clone)] pub struct PortManager { @@ -113,14 +126,18 @@ impl PortManager { #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] pub fn create_port( &self, - nic: &NetworkInterface, - source_nat: Option, - ephemeral_ip: Option, - floating_ips: &[IpAddr], - firewall_rules: &[VpcFirewallRule], - dhcp_config: DhcpCfg, - is_service: bool, + params: PortCreateParams, ) -> Result<(Port, PortTicket), Error> { + let PortCreateParams { + nic, + source_nat, + ephemeral_ip, + floating_ips, + firewall_rules, + dhcp_config, + is_service, + } = params; + let mac = *nic.mac; let vni = Vni::new(nic.vni).unwrap(); let subnet = IpNetwork::from(nic.subnet); @@ -336,16 +353,16 @@ impl PortManager { let (port, ticket) = { let mut ports = self.inner.ports.lock().unwrap(); let ticket = PortTicket::new(nic.id, nic.kind, self.inner.clone()); - let port = Port::new( - port_name.clone(), - nic.ip, + let port = Port::new(PortData { + name: port_name.clone(), + ip: nic.ip, mac, - nic.slot, + slot: nic.slot, vni, - nic.subnet, + subnet: nic.subnet, gateway, vnic, - ); + }); let old = ports.insert((nic.id, nic.kind), port.clone()); assert!( old.is_none(), diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index e43e13f96f..df5970e634 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -27,7 +27,7 @@ use backoff::BackoffError; use chrono::Utc; use illumos_utils::dladm::Etherstub; use illumos_utils::link::VnicAllocator; -use illumos_utils::opte::{DhcpCfg, PortManager}; +use illumos_utils::opte::{DhcpCfg, PortCreateParams, PortManager}; use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory}; use illumos_utils::svc::wait_for_service; use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; @@ -1323,15 +1323,15 @@ impl InstanceRunner { } else { (None, None, &[][..]) }; - let port = self.port_manager.create_port( + let port = self.port_manager.create_port(PortCreateParams { nic, - snat, + source_nat: snat, ephemeral_ip, floating_ips, - &self.firewall_rules, - self.dhcp_config.clone(), - false, - )?; + firewall_rules: &self.firewall_rules, + dhcp_config: self.dhcp_config.clone(), + is_service: false, + })?; opte_ports.push(port); } diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index 67357c9512..cc38725b61 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -3,7 +3,7 @@ use anyhow::{anyhow, Result}; use illumos_utils::dladm::Etherstub; use illumos_utils::link::VnicAllocator; use illumos_utils::opte::params::VpcFirewallRule; -use illumos_utils::opte::{DhcpCfg, PortManager}; +use illumos_utils::opte::{DhcpCfg, PortCreateParams, PortManager}; use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory}; use illumos_utils::zone::Zones; use nexus_client::types::{ProbeExternalIp, ProbeInfo}; @@ -223,12 +223,12 @@ impl ProbeManagerInner { .get(0) .ok_or(anyhow!("expected an external ip"))?; - let port = self.port_manager.create_port( - &nic, - None, - Some(eip.ip), - &[], // floating ips - &[VpcFirewallRule { + let port = self.port_manager.create_port(PortCreateParams { + nic, + source_nat: None, + ephemeral_ip: Some(eip.ip), + floating_ips: &[], + firewall_rules: &[VpcFirewallRule { status: VpcFirewallRuleStatus::Enabled, direction: VpcFirewallRuleDirection::Inbound, targets: vec![nic.clone()], @@ -238,9 +238,9 @@ impl ProbeManagerInner { action: VpcFirewallRuleAction::Allow, priority: VpcFirewallRulePriority(100), }], - DhcpCfg::default(), - false, - )?; + dhcp_config: DhcpCfg::default(), + is_service: false, + })?; let installed_zone = ZoneBuilderFactory::default() .builder() diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index b8bde2bfc8..325eb5cd61 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -49,7 +49,7 @@ use illumos_utils::dladm::{ Dladm, Etherstub, EtherstubVnic, GetSimnetError, PhysicalLink, }; use illumos_utils::link::{Link, VnicAllocator}; -use illumos_utils::opte::{DhcpCfg, Port, PortManager, PortTicket}; +use illumos_utils::opte::{DhcpCfg, Port, PortCreateParams, PortManager, PortTicket}; use illumos_utils::running_zone::{ EnsureAddressError, InstalledZone, RunCommandError, RunningZone, ZoneBuilderFactory, @@ -1266,15 +1266,15 @@ impl ServiceManager { // config allows outbound access which is enough for // Boundary NTP which needs to come up before Nexus. let port = port_manager - .create_port( + .create_port(PortCreateParams { nic, - snat, - None, + source_nat: snat, + ephemeral_ip: None, floating_ips, - &[], - DhcpCfg::default(), - true, - ) + firewall_rules: &[], + dhcp_config: DhcpCfg::default(), + is_service: true, + }) .map_err(|err| Error::ServicePortCreation { service: zone_type_str.clone(), err: Box::new(err), From 38beadd65dd3b832e4e5d382619f87a6321a7b45 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Mon, 20 May 2024 12:00:23 +0100 Subject: [PATCH 14/39] Comment adapted. --- sled-agent/src/services.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 325eb5cd61..ceaf8bec05 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1262,7 +1262,7 @@ impl ServiceManager { // Create the OPTE port for the service. // Note we don't plumb any firewall rules at this point, - // Nexus will plumb them down later but the default OPTE + // Nexus will plumb them down later but services' default OPTE // config allows outbound access which is enough for // Boundary NTP which needs to come up before Nexus. let port = port_manager From 17489cbef51bbb9b2b1113153f2fe6dcab7ee20c Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Mon, 20 May 2024 17:33:26 +0100 Subject: [PATCH 15/39] The backing code for a generational RPW Currently there are no triggers attached to most of the operations that will cause us to either a) push or b) re-resolve VPC routes, but this lays the basis for sled-agent and the background task to talk in terms of versions. --- common/src/api/internal/shared.rs | 28 ++- illumos-utils/src/opte/port_manager.rs | 62 ++++-- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/vpc_router.rs | 9 +- nexus/db-queries/src/db/datastore/vpc.rs | 251 +++++++++++++++++++++++ nexus/src/app/background/vpc_routes.rs | 128 +++++++----- openapi/sled-agent.json | 55 ++++- schema/crdb/dbinit.sql | 8 +- schema/crdb/vpc-subnet-routing/up02.sql | 7 + sled-agent/src/http_entrypoints.rs | 8 +- sled-agent/src/services.rs | 4 +- sled-agent/src/sled_agent.rs | 4 +- 12 files changed, 481 insertions(+), 84 deletions(-) diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 96f0fecd95..52601a01a8 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -611,7 +611,23 @@ pub enum RouterTarget { VpcSubnet(IpNet), } -/// XX +/// XXX +#[derive( + Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +pub struct RouterVersion { + pub router_id: Uuid, + pub generation: u64, +} + +impl RouterVersion { + pub fn is_replaced_by(&self, other: &Self) -> bool { + (self.router_id != other.router_id) + || self.generation < other.generation + } +} + +/// Implementation details on XXX #[derive( Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] @@ -620,10 +636,18 @@ pub struct RouterId { pub subnet: Option, } -/// XX +/// Version information +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +pub struct ReifiedVpcRouteState { + pub id: RouterId, + pub version: Option, +} + +/// An updated set of routes for a given #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] pub struct ReifiedVpcRouteSet { pub id: RouterId, + pub version: Option, pub routes: HashSet, } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 968854381e..d206273c48 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -19,8 +19,10 @@ use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_common::api::internal::shared::ReifiedVpcRoute; use omicron_common::api::internal::shared::ReifiedVpcRouteSet; +use omicron_common::api::internal::shared::ReifiedVpcRouteState; use omicron_common::api::internal::shared::RouterId; use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget; +use omicron_common::api::internal::shared::RouterVersion; use omicron_common::api::internal::shared::SourceNatConfig; use oxide_vpc::api::AddRouterEntryReq; use oxide_vpc::api::DelRouterEntryReq; @@ -54,6 +56,12 @@ use uuid::Uuid; // Prefix used to identify xde data links. const XDE_LINK_PREFIX: &str = "opte"; +#[derive(Debug, Clone)] +struct RouteSet { + version: Option, + routes: HashSet, +} + #[derive(Debug)] struct PortManagerInner { log: Logger, @@ -68,12 +76,11 @@ struct PortManagerInner { // (which includes the Uuid of the parent instance or service) ports: Mutex>, - // XX: vs. Hashmap? // XX: Should this be the UUID of the VPC? The rulesets are // arguably shared v4+v6, although today we don't yet // allow dual-stack, let alone v6. - // Map of all current resolved routes - routes: Mutex>>, + // Map of all current resolved routes. + routes: Mutex>, } impl PortManagerInner { @@ -379,33 +386,38 @@ impl PortManager { let system_routes = routes .entry(RouterId { vni: nic.vni, subnet: None }) .or_insert_with(|| { - let mut out = HashSet::new(); + let mut routes = HashSet::new(); + // Services do not talk to one another via OPTE, but do need // to reach out over the Internet *before* nexus is up to give // us real rules. The easiest bet is to instantiate these here. if is_service { - out.insert(ReifiedVpcRoute { + routes.insert(ReifiedVpcRoute { dest: "0.0.0.0/0".parse().unwrap(), target: ApiRouterTarget::InternetGateway, }); - out.insert(ReifiedVpcRoute { + routes.insert(ReifiedVpcRoute { dest: "::/0".parse().unwrap(), target: ApiRouterTarget::InternetGateway, }); } - out + + RouteSet { version: None, routes } }) .clone(); let custom_routes = routes .entry(RouterId { vni: nic.vni, subnet: Some(nic.subnet) }) - .or_insert_with(|| HashSet::default()); + .or_insert_with(|| RouteSet { + version: None, + routes: HashSet::default(), + }); for (class, routes) in [ (RouterClass::System, &system_routes), (RouterClass::Custom, custom_routes), ] { - for route in routes { + for route in &routes.routes { let route = AddRouterEntryReq { class, port_name: port_name.clone(), @@ -433,11 +445,11 @@ impl PortManager { Ok((port, ticket)) } - pub fn vpc_routes_list(&self) -> Vec { + pub fn vpc_routes_list(&self) -> Vec { let routes = self.inner.routes.lock().unwrap(); routes .iter() - .map(|(k, v)| ReifiedVpcRouteSet { id: *k, routes: v.clone() }) + .map(|(k, v)| ReifiedVpcRouteState { id: *k, version: v.version }) .collect() } @@ -448,19 +460,31 @@ impl PortManager { let mut routes = self.inner.routes.lock().unwrap(); let mut deltas = HashMap::new(); for set in new_routes { - let old = routes.get(&set.id); - - let (to_add, to_delete) = if let Some(old) = old { - ( - set.routes.difference(old).cloned().collect(), - old.difference(&set.routes).cloned().collect(), - ) + // We have to handle subnet router changes, as well as + // spurious updates from multiple Nexus instances. + // If there's a UUID match, only update if vers increased, + // otherwise take the update verbatim (including loss of version). + let (to_add, to_delete) = if let Some(old) = routes.get(&set.id) { + match (old.version, set.version) { + (Some(old_vers), Some(new_vers)) + if !old_vers.is_replaced_by(&new_vers) => + { + continue; + } + _ => ( + set.routes.difference(&old.routes).cloned().collect(), + old.routes.difference(&set.routes).cloned().collect(), + ), + } } else { (set.routes.clone(), HashSet::new()) }; deltas.insert(set.id, (to_add, to_delete)); - routes.insert(set.id, set.routes); + routes.insert( + set.id, + RouteSet { version: set.version, routes: set.routes }, + ); } drop(routes); diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 95d372167e..bab18d9fae 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1114,6 +1114,7 @@ table! { kind -> crate::VpcRouterKindEnum, vpc_id -> Uuid, rcgen -> Int8, + resolved_version -> Int8, } } diff --git a/nexus/db-model/src/vpc_router.rs b/nexus/db-model/src/vpc_router.rs index 71c753e6aa..51409c38d5 100644 --- a/nexus/db-model/src/vpc_router.rs +++ b/nexus/db-model/src/vpc_router.rs @@ -44,6 +44,7 @@ pub struct VpcRouter { pub vpc_id: Uuid, pub kind: VpcRouterKind, pub rcgen: Generation, + pub resolved_version: i64, } impl VpcRouter { @@ -54,7 +55,13 @@ impl VpcRouter { params: params::VpcRouterCreate, ) -> Self { let identity = VpcRouterIdentity::new(router_id, params.identity); - Self { identity, vpc_id, kind, rcgen: Generation::new() } + Self { + identity, + vpc_id, + kind, + rcgen: Generation::new(), + resolved_version: 0, + } } } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 5cbf0a5edb..69972a701c 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1696,6 +1696,7 @@ mod tests { use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; use crate::db::model::Project; use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; + use ipnetwork::Ipv4Network; use nexus_db_model::IncompleteNetworkInterface; use nexus_db_model::SledUpdate; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; @@ -2211,4 +2212,254 @@ mod tests { db.cleanup().await.unwrap(); logctx.cleanup_successful(); } + + // Test to verify that subnet CRUD operations are correctly + // reflected in the nexus-managed system router attached to a VPC. + #[tokio::test] + async fn test_vpc_system_router_sync_to_subnets() { + usdt::register_probes().unwrap(); + let logctx = + dev::test_setup_log("test_vpc_system_router_sync_to_subnets"); + let log = &logctx.log; + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // Create a project and VPC. + let project_params = params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: String::from("test project"), + }, + }; + let project = Project::new(Uuid::new_v4(), project_params); + let (authz_project, _) = datastore + .project_create(&opctx, project) + .await + .expect("failed to create project"); + + let vpc_name: external::Name = "my-vpc".parse().unwrap(); + let description = String::from("test vpc"); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: vpc_name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: vpc_name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = Vni(external::Vni::try_from(2048).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating initial VPC"; + "vni" => ?this_vni, + ); + let query = InsertVpcQuery::new(incomplete_vpc); + let (authz_vpc, db_vpc) = datastore + .project_create_vpc_raw(&opctx, &authz_project, query) + .await + .expect("failed to create initial set of VPCs") + .expect("expected an actual VPC"); + info!( + log, + "created VPC"; + "vpc" => ?db_vpc, + ); + + // Now create the system router for this VPC. Subnet CRUD + // operations need this defined to succeed. + let router = VpcRouter::new( + db_vpc.system_router_id, + db_vpc.id(), + VpcRouterKind::System, + nexus_types::external_api::params::VpcRouterCreate { + identity: IdentityMetadataCreateParams { + name: "system".parse().unwrap(), + description: description.clone(), + }, + }, + ); + + let (_, db_router) = datastore + .vpc_create_router(&opctx, &authz_vpc, router) + .await + .unwrap(); + + // InternetGateway route creation is handled by the saga proper, + // so we'll only have subnet routes here. Initially, we start with none: + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[], + ) + .await; + + // Add a new subnet and we should get a new route. + let ipv6_block = db_vpc + .ipv6_prefix + .random_subnet(external::Ipv6Net::VPC_SUBNET_IPV6_PREFIX_LENGTH) + .map(|block| block.0) + .unwrap(); + let (authz_sub0, sub0) = datastore + .vpc_create_subnet( + &opctx, + &authz_vpc, + db::model::VpcSubnet::new( + Uuid::new_v4(), + db_vpc.id(), + IdentityMetadataCreateParams { + name: "s0".parse().unwrap(), + description: "The default subnet...".into(), + }, + external::Ipv4Net( + Ipv4Network::new( + core::net::Ipv4Addr::new(172, 30, 0, 0), + 22, + ) + .unwrap(), + ), + ipv6_block, + ), + ) + .await + .unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub0], + ) + .await; + + // Add another, and get another route. + let ipv6_block = db_vpc + .ipv6_prefix + .random_subnet(external::Ipv6Net::VPC_SUBNET_IPV6_PREFIX_LENGTH) + .map(|block| block.0) + .unwrap(); + let (_, sub1) = datastore + .vpc_create_subnet( + &opctx, + &authz_vpc, + db::model::VpcSubnet::new( + Uuid::new_v4(), + db_vpc.id(), + IdentityMetadataCreateParams { + name: "s1".parse().unwrap(), + description: "A second subnet...".into(), + }, + external::Ipv4Net( + Ipv4Network::new( + core::net::Ipv4Addr::new(172, 31, 0, 0), + 22, + ) + .unwrap(), + ), + ipv6_block, + ), + ) + .await + .unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub0, &sub1], + ) + .await; + + // Rename one subnet, and our invariants should hold. + let sub0 = datastore + .vpc_update_subnet( + &opctx, + &authz_sub0, + VpcSubnetUpdate { + name: Some( + "a-new-name".parse::().unwrap().into(), + ), + description: None, + time_modified: Utc::now(), + }, + ) + .await + .unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub0, &sub1], + ) + .await; + + // Delete one, and routes should stay in sync. + datastore.vpc_delete_subnet(&opctx, &sub0, &authz_sub0).await.unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub1], + ) + .await; + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + + async fn verify_all_subnet_routes_in_router( + opctx: &OpContext, + datastore: &DataStore, + router_id: Uuid, + subnets: &[&VpcSubnet], + ) -> Vec { + let conn = datastore.pool_connection_authorized(opctx).await.unwrap(); + + use db::schema::router_route::dsl; + let routes = dsl::router_route + .filter(dsl::time_deleted.is_null()) + .filter(dsl::vpc_router_id.eq(router_id)) + .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .select(RouterRoute::as_select()) + .load_async(&*conn) + .await + .unwrap(); + + // We should have exactly as many subnet routes as subnets. + assert_eq!(routes.len(), subnets.len()); + + let mut names: HashMap<_, _> = + subnets.iter().map(|s| (s.name().clone(), 0usize)).collect(); + + // Each should have a target+dest bound to a subnet by name. + for route in &routes { + let found_name = match &route.target.0 { + RouteTarget::Subnet(name) => name, + e => panic!("found target {e:?} instead of Subnet({{name}})"), + }; + + match &route.destination.0 { + RouteDestination::Subnet(name) => assert_eq!(name, found_name), + e => panic!("found dest {e:?} instead of Subnet({{name}})"), + } + + *names.get_mut(found_name).unwrap() += 1; + } + + // Each name should be used exactly once. + for (name, count) in names { + assert_eq!(count, 1, "subnet {name} should appear exactly once") + } + + routes + } } diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index 6f21d61204..d4d6478c93 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -14,9 +14,11 @@ use nexus_types::{ deployment::SledFilter, external_api::views::SledPolicy, identity::Asset, identity::Resource, }; -use omicron_common::api::internal::shared::ReifiedVpcRoute; -use omicron_common::api::internal::shared::{ReifiedVpcRouteSet, RouterId}; +use omicron_common::api::internal::shared::{ + ReifiedVpcRoute, ReifiedVpcRouteSet, RouterId, RouterVersion, +}; use serde_json::json; +use std::collections::hash_map::Entry; use std::{ collections::{HashMap, HashSet}, sync::Arc, @@ -33,6 +35,10 @@ impl VpcRouteManager { } } +// There's a sort of eventual consistency happening here. +// ... DETAIL XX ... +// version bumps must happen AFTER other changes occur in +// children etc. to keep this sane and working. :) impl BackgroundTask for VpcRouteManager { fn activate<'a>( &'a mut self, @@ -40,7 +46,6 @@ impl BackgroundTask for VpcRouteManager { ) -> BoxFuture<'a, serde_json::Value> { async { let log = &opctx.log; - // let mut paginator = Paginator::new(MAX_SLED_AGENTS); // XX: copied from omicron#5566 let sleds = match self @@ -73,10 +78,10 @@ impl BackgroundTask for VpcRouteManager { }) .collect(); - // XX: actually reify rules. let mut known_rules: HashMap> = HashMap::new(); let mut db_routers = HashMap::new(); + let mut vni_to_vpc = HashMap::new(); for (sled, client) in sled_clients { let Ok(route_sets) = client.list_vpc_routes().await else { @@ -90,26 +95,35 @@ impl BackgroundTask for VpcRouteManager { let route_sets = route_sets.into_inner(); - // Lookup all missing (VNI, subnet) pairs we need from this sled. + // Lookup all VPC<->Subnet<->Router associations we might need, + // based on the set of VNIs reported by this sled. + // These provide the versions we'll stick with -- in the worst + // case we push newer state to a sled with an older generation + // number, which for set in &route_sets { - let system_route = - RouterId { vni: set.id.vni, subnet: None }; - - if db_routers.contains_key(&system_route) { - continue; - } - let db_vni = Vni(set.id.vni); - let Ok(vpc) = - self.datastore.resolve_vni_to_vpc(opctx, db_vni).await - else { - error!( - log, - "failed to fetch VPC from VNI"; - "sled" => sled.serial_number(), - "vni" => format!("{db_vni:?}") - ); - continue; + let maybe_vpc = vni_to_vpc.entry(set.id.vni); + let vpc = match maybe_vpc { + Entry::Occupied(_) => { + continue; + } + Entry::Vacant(v) => { + let Ok(vpc) = self + .datastore + .resolve_vni_to_vpc(opctx, db_vni) + .await + else { + error!( + log, + "failed to fetch VPC from VNI"; + "sled" => sled.serial_number(), + "vni" => ?db_vni + ); + continue; + }; + + v.insert(vpc) + } }; let vpc_id = vpc.identity().id; @@ -140,50 +154,71 @@ impl BackgroundTask for VpcRouteManager { continue; }; - db_routers.insert(system_route, system_router); - db_routers.extend(custom_routers.into_iter().map( + db_routers.insert( + RouterId { vni: set.id.vni, subnet: None }, + system_router, + ); + db_routers.extend(custom_routers.iter().map( |(subnet, router)| { ( RouterId { vni: set.id.vni, subnet: Some(subnet.ipv4_block.0.into()), }, + router.clone(), + ) + }, + )); + db_routers.extend(custom_routers.into_iter().map( + |(subnet, router)| { + ( + RouterId { + vni: set.id.vni, + subnet: Some(subnet.ipv6_block.0.into()), + }, router, ) }, )); - // XX: do this right / unify v4 and v6 - // db_routers.extend(custom_routers.into_iter().map( - // |(subnet, router)| { - // ( - // RouterId { - // vni: set.id.vni, - // subnet: Some(subnet.ipv6_block.0.into()), - // }, - // router, - // ) - // }, - // )); } - let mut to_push = HashMap::new(); + let mut to_push = Vec::new(); + let mut set_rules = |id, version, routes| { + to_push.push(ReifiedVpcRouteSet { id, routes, version }); + }; - // reify into known_rules on an as-needed basis. + // resolve into known_rules on an as-needed basis. for set in &route_sets { let Some(db_router) = db_routers.get(&set.id) else { // The sled wants to know about rules for a VPC // subnet with no custom router set. Send them - // the empty list. - to_push.insert(set.id, HashSet::new()); + // the empty list, unset its table version. + set_rules(set.id, None, HashSet::new()); continue; }; let router_id = db_router.id(); + let version = RouterVersion { + generation: db_router.resolved_version as u64, + router_id, + }; + + // Only attempt to resolve/push a ruleset if we have a different + // router ID than the sled, or a higher version number. + match &set.version { + Some(v) + if v.router_id == router_id + && v.generation >= version.generation => + { + continue; + } + _ => {} + } // We may have already resolved the rules for this - // router in a previous call. + // router in a previous iteration. if let Some(rules) = known_rules.get(&router_id) { - to_push.insert(set.id, rules.clone()); + set_rules(set.id, Some(version), rules.clone()); continue; } @@ -196,7 +231,7 @@ impl BackgroundTask for VpcRouteManager { .await { Ok(rules) => { - to_push.insert(set.id, rules.clone()); + set_rules(set.id, Some(version), rules.clone()); known_rules.insert(router_id, rules); } Err(e) => { @@ -210,17 +245,12 @@ impl BackgroundTask for VpcRouteManager { } } - let to_push = to_push - .into_iter() - .map(|(id, routes)| ReifiedVpcRouteSet { id, routes }) - .collect(); - if let Err(e) = client.set_vpc_routes(&to_push).await { error!( log, "failed to push new VPC route state from sled"; "sled" => sled.serial_number(), - "err" => format!("{e}") + "err" => ?e ); continue; }; diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index cf55e8ef74..e4778bf06a 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -955,7 +955,7 @@ }, "/vpc-routes": { "get": { - "summary": "Get the current state of VPC routing rules.", + "summary": "Get the current versions of VPC routing rules.", "operationId": "list_vpc_routes", "responses": { "200": { @@ -963,10 +963,10 @@ "content": { "application/json": { "schema": { - "title": "Array_of_ReifiedVpcRouteSet", + "title": "Array_of_ReifiedVpcRouteState", "type": "array", "items": { - "$ref": "#/components/schemas/ReifiedVpcRouteSet" + "$ref": "#/components/schemas/ReifiedVpcRouteState" } } } @@ -4326,7 +4326,7 @@ ] }, "ReifiedVpcRouteSet": { - "description": "XX", + "description": "An updated set of routes for a given", "type": "object", "properties": { "id": { @@ -4338,11 +4338,35 @@ "$ref": "#/components/schemas/ReifiedVpcRoute" }, "uniqueItems": true + }, + "version": { + "$ref": "#/components/schemas/RouterVersion" } }, "required": [ "id", - "routes" + "routes", + "version" + ] + }, + "ReifiedVpcRouteState": { + "description": "Version information", + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/RouterId" + }, + "version": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RouterVersion" + } + ] + } + }, + "required": [ + "id" ] }, "RouteConfig": { @@ -4375,7 +4399,7 @@ ] }, "RouterId": { - "description": "XX", + "description": "Implementation details on XXX", "type": "object", "properties": { "subnet": { @@ -4464,6 +4488,25 @@ } ] }, + "RouterVersion": { + "description": "XXX", + "type": "object", + "properties": { + "generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "router_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "generation", + "router_id" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 486ae68240..9466c04d15 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -1599,7 +1599,13 @@ CREATE TABLE IF NOT EXISTS omicron.public.vpc_router ( time_deleted TIMESTAMPTZ, kind omicron.public.vpc_router_kind NOT NULL, vpc_id UUID NOT NULL, - rcgen INT NOT NULL + rcgen INT NOT NULL, + /* + * version information used to trigger VPC router RPW. + * this is sensitive to CRUD on named resources beyond + * routers e.g. instances, subnets, ... + */ + resolved_version INT NOT NULL DEFAULT 0 ); CREATE UNIQUE INDEX IF NOT EXISTS lookup_router_by_vpc ON omicron.public.vpc_router ( diff --git a/schema/crdb/vpc-subnet-routing/up02.sql b/schema/crdb/vpc-subnet-routing/up02.sql index e69de29bb2..77e72961a3 100644 --- a/schema/crdb/vpc-subnet-routing/up02.sql +++ b/schema/crdb/vpc-subnet-routing/up02.sql @@ -0,0 +1,7 @@ +/* + * version information used to trigger VPC router RPW. + * this is sensitive to CRUD on named resources beyond + * routers e.g. instances, subnets, ... + */ +ALTER TABLE omicron.public.vpc_router +ADD COLUMN IF NOT EXISTS resolved_version INT NOT NULL DEFAULT 0; diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index bf85bb49a7..f8156a617e 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -33,7 +33,9 @@ use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::{ DiskRuntimeState, SledInstanceState, UpdateArtifactId, }; -use omicron_common::api::internal::shared::{ReifiedVpcRouteSet, SwitchPorts}; +use omicron_common::api::internal::shared::{ + ReifiedVpcRouteSet, ReifiedVpcRouteState, SwitchPorts, +}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_hardware::DiskVariant; @@ -1021,14 +1023,14 @@ async fn bootstore_status( Ok(HttpResponseOk(status)) } -/// Get the current state of VPC routing rules. +/// Get the current versions of VPC routing rules. #[endpoint { method = GET, path = "/vpc-routes", }] async fn list_vpc_routes( request_context: RequestContext, -) -> Result>, HttpError> { +) -> Result>, HttpError> { let sa = request_context.context(); Ok(HttpResponseOk(sa.list_vpc_routes())) } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index ceaf8bec05..a16611f9d9 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -49,7 +49,9 @@ use illumos_utils::dladm::{ Dladm, Etherstub, EtherstubVnic, GetSimnetError, PhysicalLink, }; use illumos_utils::link::{Link, VnicAllocator}; -use illumos_utils::opte::{DhcpCfg, Port, PortCreateParams, PortManager, PortTicket}; +use illumos_utils::opte::{ + DhcpCfg, Port, PortCreateParams, PortManager, PortTicket, +}; use illumos_utils::running_zone::{ EnsureAddressError, InstalledZone, RunCommandError, RunningZone, ZoneBuilderFactory, diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 263db9555e..6f73d88fe6 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -51,7 +51,7 @@ use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; use omicron_common::api::internal::shared::{ - HostPortConfig, RackNetworkConfig, ReifiedVpcRouteSet, + HostPortConfig, RackNetworkConfig, ReifiedVpcRouteSet, ReifiedVpcRouteState, }; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, @@ -1091,7 +1091,7 @@ impl SledAgent { self.inner.bootstore.clone() } - pub fn list_vpc_routes(&self) -> Vec { + pub fn list_vpc_routes(&self) -> Vec { self.inner.port_manager.vpc_routes_list() } From 006b1ca3b278b33482d06f2ada784df02cdcf11f Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 21 May 2024 11:39:07 +0100 Subject: [PATCH 16/39] Iterating. --- clients/sled-agent-client/src/lib.rs | 1 + nexus/db-queries/src/db/datastore/vpc.rs | 52 ++++++++++++++++++++++++ nexus/src/app/background/vpc_routes.rs | 11 +++-- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 58a1fc0d14..9df5230779 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -56,6 +56,7 @@ progenitor::generate_api!( ReifiedVpcRoute = omicron_common::api::internal::shared::ReifiedVpcRoute, ReifiedVpcRouteSet = omicron_common::api::internal::shared::ReifiedVpcRouteSet, RouterTarget = omicron_common::api::internal::shared::RouterTarget, + RouterVersion = omicron_common::api::internal::shared::RouterVersion, SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig, Vni = omicron_common::api::external::Vni, NetworkInterface = omicron_common::api::internal::shared::NetworkInterface, diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 69972a701c..d6b9645f93 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1684,6 +1684,58 @@ impl DataStore { Ok(out) } + + /// Trigger an RPW version bump on a single VPC router in response + /// to CRUD operations on individual routes. + pub async fn vpc_router_increment_rpw_version( + &self, + opctx: &OpContext, + authz_router: &authz::VpcRouter, + ) -> UpdateResult<()> { + opctx.authorize(authz::Action::Modify, authz_router).await?; + + use db::schema::vpc_router::dsl; + diesel::update(dsl::vpc_router) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_router.id())) + .set(dsl::resolved_version.eq(dsl::resolved_version + 1)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_router), + ) + })?; + + Ok(()) + } + + /// Trigger an RPW version bump on all routers within a VPC in + /// response to changes to named entities (e.g., subnets, instances). + pub async fn vpc_increment_rpw_version( + &self, + opctx: &OpContext, + authz_vpc: &authz::Vpc, + ) -> UpdateResult<()> { + opctx.authorize(authz::Action::CreateChild, authz_vpc).await?; + + use db::schema::vpc_router::dsl; + diesel::update(dsl::vpc_router) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::vpc_id.eq(authz_vpc.id())) + .set(dsl::resolved_version.eq(dsl::resolved_version + 1)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_vpc), + ) + })?; + + Ok(()) + } } #[cfg(test)] diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index d4d6478c93..eb19753c27 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -192,7 +192,7 @@ impl BackgroundTask for VpcRouteManager { let Some(db_router) = db_routers.get(&set.id) else { // The sled wants to know about rules for a VPC // subnet with no custom router set. Send them - // the empty list, unset its table version. + // the empty list, and unset its table version. set_rules(set.id, None, HashSet::new()); continue; }; @@ -203,12 +203,11 @@ impl BackgroundTask for VpcRouteManager { router_id, }; - // Only attempt to resolve/push a ruleset if we have a different - // router ID than the sled, or a higher version number. + // Only attempt to resolve/push a ruleset if we have a + // different router ID than the sled, or a higher version + // number. match &set.version { - Some(v) - if v.router_id == router_id - && v.generation >= version.generation => + Some(v) if !v.is_replaced_by(&version) => { continue; } From c7de875d028d1e4bd19c602e20d7a97f643bf221 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 00:43:37 +0100 Subject: [PATCH 17/39] Trigger RPW in all the right spots. --- clients/sled-agent-client/src/lib.rs | 2 +- .../src/db/datastore/network_interface.rs | 20 +++++- nexus/db-queries/src/db/datastore/vpc.rs | 35 ++++------ nexus/src/app/background/vpc_routes.rs | 23 +++---- nexus/src/app/instance.rs | 1 + nexus/src/app/vpc_router.rs | 64 ++++++++++++++++--- nexus/src/app/vpc_subnet.rs | 31 ++++++--- openapi/sled-agent.json | 10 ++- sled-agent/src/probe_manager.rs | 28 +++++++- 9 files changed, 155 insertions(+), 59 deletions(-) diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 9df5230779..d4fb36004f 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -35,7 +35,7 @@ progenitor::generate_api!( PortConfigV1 = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, RouteConfig = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, IpNet = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, - OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] } + OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] }, RouterId = { derives = [PartialEq, Eq, Hash, Debug, Deserialize, Serialize] }, }, //TODO trade the manual transformations later in this file for the diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index b426c8b472..2ac1f531a3 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -137,11 +137,27 @@ impl DataStore { ), )); } - self.create_network_interface_raw(opctx, interface) + + let out = self + .create_network_interface_raw(opctx, interface) .await // Convert to `InstanceNetworkInterface` before returning; we know // this is valid as we've checked the condition on-entry. - .map(NetworkInterface::as_instance) + .map(NetworkInterface::as_instance)?; + + // `instance:xxx` targets in router rules resolve to the primary + // NIC of that instance. Accordingly, NIC create may cause dangling + // entries to re-resolve to a valid instance (even if it is not yet + // started). + // This will not trigger the RPW directly, we still need to do so + // in e.g. the instance watcher task. + if out.primary { + self.vpc_increment_rpw_version(opctx, out.vpc_id) + .await + .map_err(|e| network_interface::InsertError::External(e))?; + } + + Ok(out) } /// List network interfaces associated with a given service. diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index d6b9645f93..fd90c8a7e6 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1409,7 +1409,9 @@ impl DataStore { Ok(()) }}).await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + self.vpc_increment_rpw_version(opctx, vpc_id).await } /// Look up a VPC by VNI. @@ -1690,49 +1692,40 @@ impl DataStore { pub async fn vpc_router_increment_rpw_version( &self, opctx: &OpContext, - authz_router: &authz::VpcRouter, + router_id: Uuid, ) -> UpdateResult<()> { - opctx.authorize(authz::Action::Modify, authz_router).await?; + // NOTE: this operation and `vpc_increment_rpw_version` do not + // have auth checks, as these can occur in connection with unrelated + // resources -- the current user may have access to those, but be unable + // to modify the entire set of VPC routers in a project. use db::schema::vpc_router::dsl; diesel::update(dsl::vpc_router) .filter(dsl::time_deleted.is_null()) - .filter(dsl::id.eq(authz_router.id())) + .filter(dsl::id.eq(router_id)) .set(dsl::resolved_version.eq(dsl::resolved_version + 1)) .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_router), - ) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; Ok(()) } - /// Trigger an RPW version bump on all routers within a VPC in + /// Trigger an RPW version bump on *all* routers within a VPC in /// response to changes to named entities (e.g., subnets, instances). pub async fn vpc_increment_rpw_version( &self, opctx: &OpContext, - authz_vpc: &authz::Vpc, + vpc_id: Uuid, ) -> UpdateResult<()> { - opctx.authorize(authz::Action::CreateChild, authz_vpc).await?; - use db::schema::vpc_router::dsl; diesel::update(dsl::vpc_router) .filter(dsl::time_deleted.is_null()) - .filter(dsl::vpc_id.eq(authz_vpc.id())) + .filter(dsl::vpc_id.eq(vpc_id)) .set(dsl::resolved_version.eq(dsl::resolved_version + 1)) .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_vpc), - ) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; Ok(()) } diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index eb19753c27..6cdf720ce2 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -207,8 +207,7 @@ impl BackgroundTask for VpcRouteManager { // different router ID than the sled, or a higher version // number. match &set.version { - Some(v) if !v.is_replaced_by(&version) => - { + Some(v) if !v.is_replaced_by(&version) => { continue; } _ => {} @@ -244,15 +243,17 @@ impl BackgroundTask for VpcRouteManager { } } - if let Err(e) = client.set_vpc_routes(&to_push).await { - error!( - log, - "failed to push new VPC route state from sled"; - "sled" => sled.serial_number(), - "err" => ?e - ); - continue; - }; + if !to_push.is_empty() { + if let Err(e) = client.set_vpc_routes(&to_push).await { + error!( + log, + "failed to push new VPC route state from sled"; + "sled" => sled.serial_number(), + "err" => ?e + ); + continue; + }; + } } json!({}) diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 50b46c8e8d..090caddf18 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1524,6 +1524,7 @@ impl super::Nexus { new_runtime_state, ) .await?; + self.vpc_needed_notify_sleds(); Ok(()) } diff --git a/nexus/src/app/vpc_router.rs b/nexus/src/app/vpc_router.rs index e65b2a8605..ae2fdffeeb 100644 --- a/nexus/src/app/vpc_router.rs +++ b/nexus/src/app/vpc_router.rs @@ -83,6 +83,10 @@ impl super::Nexus { .db_datastore .vpc_create_router(&opctx, &authz_vpc, router) .await?; + + // Note: we don't trigger the route RPW here as it's impossible + // for the router to be bound to a subnet at this point. + Ok(router) } @@ -114,8 +118,8 @@ impl super::Nexus { .await } - // TODO: When a router is deleted it should be unassociated w/ any subnets it may be associated with - // or trigger an error + // TODO(now): When a router is deleted it should be unassociated w/ any subnets it may be associated with + // or trigger an error. pub(crate) async fn vpc_delete_router( &self, opctx: &OpContext, @@ -130,7 +134,12 @@ impl super::Nexus { if db_router.kind == VpcRouterKind::System { return Err(Error::invalid_request("Cannot delete system router")); } - self.db_datastore.vpc_delete_router(opctx, &authz_router).await + let out = + self.db_datastore.vpc_delete_router(opctx, &authz_router).await?; + + self.vpc_needed_notify_sleds(); + + Ok(out) } // Routes @@ -197,6 +206,9 @@ impl super::Nexus { .db_datastore .router_create_route(&opctx, &authz_router, route) .await?; + + self.vpc_router_increment_rpw_version(opctx, &authz_router).await?; + Ok(route) } @@ -219,7 +231,7 @@ impl super::Nexus { route_lookup: &lookup::RouterRoute<'_>, params: ¶ms::RouterRouteUpdate, ) -> UpdateResult { - let (.., vpc, _, authz_route, db_route) = + let (.., vpc, authz_router, authz_route, db_route) = route_lookup.fetch_for(authz::Action::Modify).await?; // TODO: Write a test for this once there's a way to test it (i.e. // subnets automatically register to the system router table) @@ -234,9 +246,14 @@ impl super::Nexus { ))); } } - self.db_datastore + let out = self + .db_datastore .router_update_route(&opctx, &authz_route, params.clone().into()) - .await + .await?; + + self.vpc_router_increment_rpw_version(opctx, &authz_router).await?; + + Ok(out) } pub(crate) async fn router_delete_route( @@ -244,7 +261,7 @@ impl super::Nexus { opctx: &OpContext, route_lookup: &lookup::RouterRoute<'_>, ) -> DeleteResult { - let (.., authz_route, db_route) = + let (.., authz_router, authz_route, db_route) = route_lookup.fetch_for(authz::Action::Delete).await?; // Only custom routes can be deleted @@ -254,6 +271,37 @@ impl super::Nexus { "DELETE not allowed on system routes", )); } - self.db_datastore.router_delete_route(opctx, &authz_route).await + let out = + self.db_datastore.router_delete_route(opctx, &authz_route).await?; + + self.vpc_router_increment_rpw_version(opctx, &authz_router).await?; + + Ok(out) + } + + /// Trigger the VPC routing RPW in repsonse to a state change + /// or a new possible listener (e.g., instance/probe start, NIC + /// create). + pub(crate) fn vpc_needed_notify_sleds(&self) { + self.background_tasks + .activate(&self.background_tasks.task_vpc_route_manager) + } + + /// Trigger an RPW version bump on a single VPC router in response + /// to CRUD operations on individual routes. + /// + /// This will also awaken the VPC Router RPW. + pub(crate) async fn vpc_router_increment_rpw_version( + &self, + opctx: &OpContext, + authz_router: &authz::VpcRouter, + ) -> UpdateResult<()> { + self.datastore() + .vpc_router_increment_rpw_version(opctx, authz_router.id()) + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(()) } } diff --git a/nexus/src/app/vpc_subnet.rs b/nexus/src/app/vpc_subnet.rs index 4c5a569201..0e3affb470 100644 --- a/nexus/src/app/vpc_subnet.rs +++ b/nexus/src/app/vpc_subnet.rs @@ -63,8 +63,7 @@ impl super::Nexus { )), } } - // TODO: When a subnet is created it should add a route entry into the VPC's - // system router + pub(crate) async fn vpc_create_subnet( &self, opctx: &OpContext, @@ -108,7 +107,7 @@ impl super::Nexus { // See for // details. let subnet_id = Uuid::new_v4(); - match params.ipv6_block { + let out = match params.ipv6_block { None => { const NUM_RETRIES: usize = 2; let mut retry = 0; @@ -212,7 +211,11 @@ impl super::Nexus { .map(|(.., subnet)| subnet) .map_err(SubnetError::into_external) } - } + }?; + + self.vpc_needed_notify_sleds(); + + Ok(out) } pub(crate) async fn vpc_subnet_list( @@ -234,13 +237,16 @@ impl super::Nexus { ) -> UpdateResult { let (.., authz_subnet) = vpc_subnet_lookup.lookup_for(authz::Action::Modify).await?; - self.db_datastore + let out = self + .db_datastore .vpc_update_subnet(&opctx, &authz_subnet, params.clone().into()) - .await + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(out) } - // TODO: When a subnet is deleted it should remove its entry from the VPC's - // system router. pub(crate) async fn vpc_delete_subnet( &self, opctx: &OpContext, @@ -248,9 +254,14 @@ impl super::Nexus { ) -> DeleteResult { let (.., authz_subnet, db_subnet) = vpc_subnet_lookup.fetch_for(authz::Action::Delete).await?; - self.db_datastore + let out = self + .db_datastore .vpc_delete_subnet(opctx, &db_subnet, &authz_subnet) - .await + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(out) } pub(crate) async fn subnet_list_instance_network_interfaces( diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index e4778bf06a..8c71d9eeb1 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -4340,13 +4340,17 @@ "uniqueItems": true }, "version": { - "$ref": "#/components/schemas/RouterVersion" + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RouterVersion" + } + ] } }, "required": [ "id", - "routes", - "version" + "routes" ] }, "ReifiedVpcRouteState": { diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index cc38725b61..eabf3850af 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -6,7 +6,9 @@ use illumos_utils::opte::params::VpcFirewallRule; use illumos_utils::opte::{DhcpCfg, PortCreateParams, PortManager}; use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory}; use illumos_utils::zone::Zones; -use nexus_client::types::{ProbeExternalIp, ProbeInfo}; +use nexus_client::types::{ + BackgroundTasksActivateRequest, ProbeExternalIp, ProbeInfo, +}; use omicron_common::api::external::{ VpcFirewallRuleAction, VpcFirewallRuleDirection, VpcFirewallRulePriority, VpcFirewallRuleStatus, @@ -179,24 +181,44 @@ impl ProbeManagerInner { } }; - self.add(target.difference(¤t)).await; + let n_added = self.add(target.difference(¤t)).await; self.remove(current.difference(&target)).await; self.check(current.intersection(&target)).await; + + // If we have created some new probes, we may (in future) need the control plane + // to provide us with valid routes for the VPC the probe belongs to. + if n_added > 0 { + if let Err(e) = self + .nexus_client + .client() + .bgtask_activate(&BackgroundTasksActivateRequest { + bgtask_names: vec!["vpc_route_manager".into()], + }) + .await + { + error!(self.log, "get routes for probe: {e}"); + } + } } }) } /// Add a set of probes to this sled. - async fn add<'a, I>(self: &Arc, probes: I) + /// + /// Returns the number of inserted probes. + async fn add<'a, I>(self: &Arc, probes: I) -> usize where I: Iterator, { + let mut i = 0; for probe in probes { info!(self.log, "adding probe {}", probe.id); if let Err(e) = self.add_probe(probe).await { error!(self.log, "add probe: {e}"); } + i += 1; } + i } /// Add a probe to this sled. This sets up resources for the probe zone From 40edbc8faf85a79a864a3b18d5c7e304b293f846 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 11:12:20 +0100 Subject: [PATCH 18/39] Unpublish VPC routers API. We'll get to that in the next PR. --- nexus/src/external_api/http_entrypoints.rs | 10 + nexus/tests/output/nexus_tags.txt | 10 - openapi/nexus.json | 1663 ++++---------------- 3 files changed, 289 insertions(+), 1394 deletions(-) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 2678768b48..e814df2b61 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -5446,6 +5446,7 @@ async fn vpc_firewall_rules_update( method = GET, path = "/v1/vpc-routers", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_list( rqctx: RequestContext, @@ -5485,6 +5486,7 @@ async fn vpc_router_list( method = GET, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_view( rqctx: RequestContext, @@ -5518,6 +5520,7 @@ async fn vpc_router_view( method = POST, path = "/v1/vpc-routers", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_create( rqctx: RequestContext, @@ -5553,6 +5556,7 @@ async fn vpc_router_create( method = DELETE, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_delete( rqctx: RequestContext, @@ -5586,6 +5590,7 @@ async fn vpc_router_delete( method = PUT, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_update( rqctx: RequestContext, @@ -5625,6 +5630,7 @@ async fn vpc_router_update( method = GET, path = "/v1/vpc-router-routes", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_list( rqctx: RequestContext, @@ -5666,6 +5672,7 @@ async fn vpc_router_route_list( method = GET, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_view( rqctx: RequestContext, @@ -5702,6 +5709,7 @@ async fn vpc_router_route_view( method = POST, path = "/v1/vpc-router-routes", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_create( rqctx: RequestContext, @@ -5737,6 +5745,7 @@ async fn vpc_router_route_create( method = DELETE, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_delete( rqctx: RequestContext, @@ -5772,6 +5781,7 @@ async fn vpc_router_route_delete( method = PUT, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_update( rqctx: RequestContext, diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 35d8c32561..a32fe5c4b9 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -232,16 +232,6 @@ vpc_delete DELETE /v1/vpcs/{vpc} vpc_firewall_rules_update PUT /v1/vpc-firewall-rules vpc_firewall_rules_view GET /v1/vpc-firewall-rules vpc_list GET /v1/vpcs -vpc_router_create POST /v1/vpc-routers -vpc_router_delete DELETE /v1/vpc-routers/{router} -vpc_router_list GET /v1/vpc-routers -vpc_router_route_create POST /v1/vpc-router-routes -vpc_router_route_delete DELETE /v1/vpc-router-routes/{route} -vpc_router_route_list GET /v1/vpc-router-routes -vpc_router_route_update PUT /v1/vpc-router-routes/{route} -vpc_router_route_view GET /v1/vpc-router-routes/{route} -vpc_router_update PUT /v1/vpc-routers/{router} -vpc_router_view GET /v1/vpc-routers/{router} vpc_subnet_create POST /v1/vpc-subnets vpc_subnet_delete DELETE /v1/vpc-subnets/{subnet} vpc_subnet_list GET /v1/vpc-subnets diff --git a/openapi/nexus.json b/openapi/nexus.json index 55f83f4a24..92af2a6b74 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -8346,14 +8346,13 @@ } } }, - "/v1/vpc-router-routes": { + "/v1/vpc-subnets": { "get": { "tags": [ "vpcs" ], - "summary": "List routes", - "description": "List the routes associated with a router in a particular VPC.", - "operationId": "vpc_router_route_list", + "summary": "List subnets", + "operationId": "vpc_subnet_list", "parameters": [ { "in": "query", @@ -8383,14 +8382,6 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "sort_by", @@ -8401,7 +8392,7 @@ { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8413,7 +8404,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteResultsPage" + "$ref": "#/components/schemas/VpcSubnetResultsPage" } } } @@ -8427,7 +8418,7 @@ }, "x-dropshot-pagination": { "required": [ - "router" + "vpc" ] } }, @@ -8435,8 +8426,8 @@ "tags": [ "vpcs" ], - "summary": "Create route", - "operationId": "vpc_router_route_create", + "summary": "Create subnet", + "operationId": "vpc_subnet_create", "parameters": [ { "in": "query", @@ -8446,19 +8437,11 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8468,7 +8451,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteCreate" + "$ref": "#/components/schemas/VpcSubnetCreate" } } }, @@ -8480,7 +8463,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -8494,18 +8477,18 @@ } } }, - "/v1/vpc-router-routes/{route}": { + "/v1/vpc-subnets/{subnet}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch route", - "operationId": "vpc_router_route_view", + "summary": "Fetch subnet", + "operationId": "vpc_subnet_view", "parameters": [ { "in": "path", - "name": "route", - "description": "Name or ID of the route", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8519,19 +8502,10 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8543,7 +8517,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -8560,13 +8534,13 @@ "tags": [ "vpcs" ], - "summary": "Update route", - "operationId": "vpc_router_route_update", + "summary": "Update subnet", + "operationId": "vpc_subnet_update", "parameters": [ { "in": "path", - "name": "route", - "description": "Name or ID of the route", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8580,18 +8554,10 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8601,7 +8567,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteUpdate" + "$ref": "#/components/schemas/VpcSubnetUpdate" } } }, @@ -8613,7 +8579,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -8630,13 +8596,13 @@ "tags": [ "vpcs" ], - "summary": "Delete route", - "operationId": "vpc_router_route_delete", + "summary": "Delete subnet", + "operationId": "vpc_subnet_delete", "parameters": [ { "in": "path", - "name": "route", - "description": "Name or ID of the route", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8650,18 +8616,10 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8680,14 +8638,23 @@ } } }, - "/v1/vpc-routers": { + "/v1/vpc-subnets/{subnet}/network-interfaces": { "get": { "tags": [ "vpcs" ], - "summary": "List routers", - "operationId": "vpc_router_list", + "summary": "List network interfaces", + "operationId": "vpc_subnet_list_network_interfaces", "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "limit", @@ -8738,7 +8705,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterResultsPage" + "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" } } } @@ -8751,30 +8718,89 @@ } }, "x-dropshot-pagination": { - "required": [ - "vpc" - ] + "required": [] } - }, - "post": { + } + }, + "/v1/vpcs": { + "get": { "tags": [ "vpcs" ], - "summary": "Create VPC router", - "operationId": "vpc_router_create", + "summary": "List VPCs", + "operationId": "vpc_list", "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create VPC", + "operationId": "vpc_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8785,7 +8811,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterCreate" + "$ref": "#/components/schemas/VpcCreate" } } }, @@ -8797,7 +8823,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/Vpc" } } } @@ -8811,18 +8837,18 @@ } } }, - "/v1/vpc-routers/{router}": { + "/v1/vpcs/{vpc}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch router", - "operationId": "vpc_router_view", + "summary": "Fetch VPC", + "operationId": "vpc_view", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8831,15 +8857,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8851,7 +8869,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/Vpc" } } } @@ -8868,13 +8886,13 @@ "tags": [ "vpcs" ], - "summary": "Update router", - "operationId": "vpc_router_update", + "summary": "Update a VPC", + "operationId": "vpc_update", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8883,15 +8901,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8901,7 +8911,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterUpdate" + "$ref": "#/components/schemas/VpcUpdate" } } }, @@ -8913,7 +8923,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/Vpc" } } } @@ -8930,13 +8940,13 @@ "tags": [ "vpcs" ], - "summary": "Delete router", - "operationId": "vpc_router_delete", + "summary": "Delete VPC", + "operationId": "vpc_delete", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8945,15 +8955,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8971,649 +8973,21 @@ } } } - }, - "/v1/vpc-subnets": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List subnets", - "operationId": "vpc_subnet_list", - "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnetResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "vpc" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create subnet", - "operationId": "vpc_subnet_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnetCreate" - } - } - }, - "required": true - }, - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnet" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/vpc-subnets/{subnet}": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "Fetch subnet", - "operationId": "vpc_subnet_view", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnet" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "tags": [ - "vpcs" - ], - "summary": "Update subnet", - "operationId": "vpc_subnet_update", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnetUpdate" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnet" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "vpcs" - ], - "summary": "Delete subnet", - "operationId": "vpc_subnet_delete", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/vpc-subnets/{subnet}/network-interfaces": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List network interfaces", - "operationId": "vpc_subnet_list_network_interfaces", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [] - } - } - }, - "/v1/vpcs": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List VPCs", - "operationId": "vpc_list", - "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "project" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create VPC", - "operationId": "vpc_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcCreate" - } - } - }, - "required": true - }, - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Vpc" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/vpcs/{vpc}": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "Fetch VPC", - "operationId": "vpc_view", - "parameters": [ - { - "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Vpc" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "tags": [ - "vpcs" - ], - "summary": "Update a VPC", - "operationId": "vpc_update", - "parameters": [ - { - "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcUpdate" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Vpc" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "vpcs" - ], - "summary": "Delete VPC", - "operationId": "vpc_delete", - "parameters": [ - { - "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - } - }, - "components": { - "schemas": { - "Address": { - "description": "An address tied to an address lot.", - "type": "object", - "properties": { - "address": { - "description": "The address and prefix length of this address.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] + } + }, + "components": { + "schemas": { + "Address": { + "description": "An address tied to an address lot.", + "type": "object", + "properties": { + "address": { + "description": "The address and prefix length of this address.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] }, "address_lot": { "description": "The address lot this address is drawn from.", @@ -16233,179 +15607,32 @@ }, "time_created": { "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - } - }, - "required": [ - "description", - "id", - "name", - "time_created", - "time_modified" - ] - }, - "ProjectCreate": { - "description": "Create-time parameters for a `Project`", - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "name": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "description", - "name" - ] - }, - "ProjectResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/Project" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" - ] - }, - "ProjectRole": { - "type": "string", - "enum": [ - "admin", - "collaborator", - "viewer" - ] - }, - "ProjectRolePolicy": { - "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", - "type": "object", - "properties": { - "role_assignments": { - "description": "Roles directly assigned on this resource", - "type": "array", - "items": { - "$ref": "#/components/schemas/ProjectRoleRoleAssignment" - } - } - }, - "required": [ - "role_assignments" - ] - }, - "ProjectRoleRoleAssignment": { - "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", - "type": "object", - "properties": { - "identity_id": { - "type": "string", - "format": "uuid" - }, - "identity_type": { - "$ref": "#/components/schemas/IdentityType" - }, - "role_name": { - "$ref": "#/components/schemas/ProjectRole" - } - }, - "required": [ - "identity_id", - "identity_type", - "role_name" - ] - }, - "ProjectUpdate": { - "description": "Updateable properties of a `Project`", - "type": "object", - "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "name": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - } - } - }, - "Rack": { - "description": "View of an Rack", - "type": "object", - "properties": { - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "time_created": { - "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - } - }, - "required": [ - "id", - "time_created", - "time_modified" - ] - }, - "RackResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/Rack" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" } }, "required": [ - "items" + "description", + "id", + "name", + "time_created", + "time_modified" ] }, - "Role": { - "description": "View of a Role", + "ProjectCreate": { + "description": "Create-time parameters for a `Project`", "type": "object", "properties": { "description": { "type": "string" }, "name": { - "$ref": "#/components/schemas/RoleName" + "$ref": "#/components/schemas/Name" } }, "required": [ @@ -16413,14 +15640,7 @@ "name" ] }, - "RoleName": { - "title": "A name for a built-in role", - "description": "Role names consist of two string components separated by dot (\".\").", - "type": "string", - "pattern": "[a-z-]+\\.[a-z-]+", - "maxLength": 63 - }, - "RoleResultsPage": { + "ProjectResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -16428,7 +15648,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/Role" + "$ref": "#/components/schemas/Project" } }, "next_page": { @@ -16441,284 +15661,77 @@ "items" ] }, - "Route": { - "description": "A route to a destination network through a gateway address.", - "type": "object", - "properties": { - "dst": { - "description": "The route destination.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] - }, - "gw": { - "description": "The route gateway.", - "type": "string", - "format": "ip" - }, - "vid": { - "nullable": true, - "description": "VLAN id the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "dst", - "gw" + "ProjectRole": { + "type": "string", + "enum": [ + "admin", + "collaborator", + "viewer" ] }, - "RouteConfig": { - "description": "Route configuration data associated with a switch port configuration.", + "ProjectRolePolicy": { + "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", "type": "object", "properties": { - "routes": { - "description": "The set of routes assigned to a switch port.", + "role_assignments": { + "description": "Roles directly assigned on this resource", "type": "array", "items": { - "$ref": "#/components/schemas/Route" + "$ref": "#/components/schemas/ProjectRoleRoleAssignment" } } }, "required": [ - "routes" - ] - }, - "RouteDestination": { - "description": "A `RouteDestination` is used to match traffic with a routing rule, on the destination of that traffic.\n\nWhen traffic is to be sent to a destination that is within a given `RouteDestination`, the corresponding `RouterRoute` applies, and traffic will be forward to the `RouteTarget` for that rule.", - "oneOf": [ - { - "description": "Route applies to traffic destined for a specific IP address", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ip" - ] - }, - "value": { - "type": "string", - "format": "ip" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Route applies to traffic destined for a specific IP subnet", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ip_net" - ] - }, - "value": { - "$ref": "#/components/schemas/IpNet" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Route applies to traffic destined for the given VPC.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "vpc" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Route applies to traffic", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "subnet" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - } + "role_assignments" ] }, - "RouteTarget": { - "description": "A `RouteTarget` describes the possible locations that traffic matching a route destination can be sent.", - "oneOf": [ - { - "description": "Forward traffic to a particular IP address.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ip" - ] - }, - "value": { - "type": "string", - "format": "ip" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to a VPC", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "vpc" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to a VPC Subnet", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "subnet" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to a specific instance", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "instance" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to an internet gateway", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "internet_gateway" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Drop matching traffic", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "drop" - ] - } - }, - "required": [ - "type" - ] + "ProjectRoleRoleAssignment": { + "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", + "type": "object", + "properties": { + "identity_id": { + "type": "string", + "format": "uuid" + }, + "identity_type": { + "$ref": "#/components/schemas/IdentityType" + }, + "role_name": { + "$ref": "#/components/schemas/ProjectRole" } + }, + "required": [ + "identity_id", + "identity_type", + "role_name" ] }, - "RouterRoute": { - "description": "A route defines a rule that governs where traffic should be sent based on its destination.", + "ProjectUpdate": { + "description": "Updateable properties of a `Project`", "type": "object", "properties": { "description": { - "description": "human-readable free-form text about a resource", + "nullable": true, "type": "string" }, - "destination": { - "$ref": "#/components/schemas/RouteDestination" - }, - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "kind": { - "description": "Describes the kind of router. Set at creation. `read-only`", - "allOf": [ - { - "$ref": "#/components/schemas/RouterRouteKind" - } - ] - }, "name": { - "description": "unique, mutable, user-controlled identifier for each resource", + "nullable": true, "allOf": [ { "$ref": "#/components/schemas/Name" } ] - }, - "target": { - "$ref": "#/components/schemas/RouteTarget" + } + } + }, + "Rack": { + "description": "View of an Rack", + "type": "object", + "properties": { + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" }, "time_created": { "description": "timestamp when this resource was created", @@ -16729,83 +15742,59 @@ "description": "timestamp when this resource was last modified", "type": "string", "format": "date-time" - }, - "vpc_router_id": { - "description": "The ID of the VPC Router to which the route belongs", - "type": "string", - "format": "uuid" } }, "required": [ - "description", - "destination", "id", - "kind", - "name", - "target", "time_created", - "time_modified", - "vpc_router_id" + "time_modified" + ] + }, + "RackResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Rack" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" ] }, - "RouterRouteCreate": { - "description": "Create-time parameters for a `RouterRoute`", + "Role": { + "description": "View of a Role", "type": "object", "properties": { "description": { "type": "string" }, - "destination": { - "$ref": "#/components/schemas/RouteDestination" - }, "name": { - "$ref": "#/components/schemas/Name" - }, - "target": { - "$ref": "#/components/schemas/RouteTarget" + "$ref": "#/components/schemas/RoleName" } }, "required": [ "description", - "destination", - "name", - "target" + "name" ] }, - "RouterRouteKind": { - "description": "The kind of a `RouterRoute`\n\nThe kind determines certain attributes such as if the route is modifiable and describes how or where the route was created.", - "oneOf": [ - { - "description": "Determines the default destination of traffic, such as whether it goes to the internet or not.\n\n`Destination: An Internet Gateway` `Modifiable: true`", - "type": "string", - "enum": [ - "default" - ] - }, - { - "description": "Automatically added for each VPC Subnet in the VPC\n\n`Destination: A VPC Subnet` `Modifiable: false`", - "type": "string", - "enum": [ - "vpc_subnet" - ] - }, - { - "description": "Automatically added when VPC peering is established\n\n`Destination: A different VPC` `Modifiable: false`", - "type": "string", - "enum": [ - "vpc_peering" - ] - }, - { - "description": "Created by a user; see `RouteTarget`\n\n`Destination: User defined` `Modifiable: true`", - "type": "string", - "enum": [ - "custom" - ] - } - ] + "RoleName": { + "title": "A name for a built-in role", + "description": "Role names consist of two string components separated by dot (\".\").", + "type": "string", + "pattern": "[a-z-]+\\.[a-z-]+", + "maxLength": 63 }, - "RouterRouteResultsPage": { + "RoleResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -16813,7 +15802,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/Role" } }, "next_page": { @@ -16826,32 +15815,50 @@ "items" ] }, - "RouterRouteUpdate": { - "description": "Updateable properties of a `RouterRoute`", + "Route": { + "description": "A route to a destination network through a gateway address.", "type": "object", "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "destination": { - "$ref": "#/components/schemas/RouteDestination" - }, - "name": { - "nullable": true, + "dst": { + "description": "The route destination.", "allOf": [ { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/IpNet" } ] }, - "target": { - "$ref": "#/components/schemas/RouteTarget" + "gw": { + "description": "The route gateway.", + "type": "string", + "format": "ip" + }, + "vid": { + "nullable": true, + "description": "VLAN id the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "dst", + "gw" + ] + }, + "RouteConfig": { + "description": "Route configuration data associated with a switch port configuration.", + "type": "object", + "properties": { + "routes": { + "description": "The set of routes assigned to a switch port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Route" + } } }, "required": [ - "destination", - "target" + "routes" ] }, "SamlIdentityProvider": { @@ -19949,118 +18956,6 @@ "items" ] }, - "VpcRouter": { - "description": "A VPC router defines a series of rules that indicate where traffic should be sent depending on its destination.", - "type": "object", - "properties": { - "description": { - "description": "human-readable free-form text about a resource", - "type": "string" - }, - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "kind": { - "$ref": "#/components/schemas/VpcRouterKind" - }, - "name": { - "description": "unique, mutable, user-controlled identifier for each resource", - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - }, - "time_created": { - "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - }, - "vpc_id": { - "description": "The VPC to which the router belongs.", - "type": "string", - "format": "uuid" - } - }, - "required": [ - "description", - "id", - "kind", - "name", - "time_created", - "time_modified", - "vpc_id" - ] - }, - "VpcRouterCreate": { - "description": "Create-time parameters for a `VpcRouter`", - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "name": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "description", - "name" - ] - }, - "VpcRouterKind": { - "type": "string", - "enum": [ - "system", - "custom" - ] - }, - "VpcRouterResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/VpcRouter" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" - ] - }, - "VpcRouterUpdate": { - "description": "Updateable properties of a `VpcRouter`", - "type": "object", - "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "name": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - } - } - }, "VpcSubnet": { "description": "A VPC subnet represents a logical grouping for instances that allows network traffic between them, within a IPv4 subnetwork or optionally an IPv6 subnetwork.", "type": "object", From 3b3abb1d1b89adef7d9a1202aacac607e320e4d9 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 16:45:06 +0100 Subject: [PATCH 19/39] Fixup broken tests. --- dev-tools/omdb/tests/env.out | 12 ++++ dev-tools/omdb/tests/successes.out | 13 +++- dev-tools/preprocessed_configs/config.xml | 41 ++++++++++++ nexus/src/app/sagas/vpc_create.rs | 17 ++++- .../tests/integration_tests/router_routes.rs | 65 +++++++++++++------ 5 files changed, 124 insertions(+), 24 deletions(-) create mode 100644 dev-tools/preprocessed_configs/config.xml diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 5716510602..b36280980e 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -114,6 +114,10 @@ task: "switch_port_config_manager" manages switch port settings for rack switches +task: "vpc_route_manager" + propagates updated VPC routes to all OPTE ports + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT @@ -225,6 +229,10 @@ task: "switch_port_config_manager" manages switch port settings for rack switches +task: "vpc_route_manager" + propagates updated VPC routes to all OPTE ports + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. @@ -323,6 +331,10 @@ task: "switch_port_config_manager" manages switch port settings for rack switches +task: "vpc_route_manager" + propagates updated VPC routes to all OPTE ports + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index c4c28460b8..07960112d6 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -291,6 +291,10 @@ task: "switch_port_config_manager" manages switch port settings for rack switches +task: "vpc_route_manager" + propagates updated VPC routes to all OPTE ports + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ @@ -426,7 +430,7 @@ task: "metrics_producer_gc" currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms -warning: unknown background task: "metrics_producer_gc" (don't know how to interpret details: Object {"expiration": String(""), "pruned": Array []}) +warning: unknown background task: "metrics_producer_gc" (don't know how to interpret details: Object {"expiration": String(""), "pruned": Array []}) task: "phantom_disks" configured period: every 30s @@ -471,6 +475,13 @@ task: "switch_port_config_manager" started at (s ago) and ran for ms warning: unknown background task: "switch_port_config_manager" (don't know how to interpret details: Object {}) +task: "vpc_route_manager" + configured period: every 30s + currently executing: no + last completed activation: , triggered by an explicit signal + started at (s ago) and ran for ms +warning: unknown background task: "vpc_route_manager" (don't know how to interpret details: Object {}) + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ diff --git a/dev-tools/preprocessed_configs/config.xml b/dev-tools/preprocessed_configs/config.xml new file mode 100644 index 0000000000..9b13f12aea --- /dev/null +++ b/dev-tools/preprocessed_configs/config.xml @@ -0,0 +1,41 @@ + + + + + trace + true + + + 8123 + 9000 + 9004 + + ./ + + true + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + \ No newline at end of file diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index cc62d9315d..c84cf5ff20 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -587,12 +587,25 @@ pub(crate) mod test { .await .expect("Failed to delete default Subnet"); - // Default route + // Default gateway routes let (.., authz_route, _route) = LookupPath::new(&opctx, &datastore) .project_id(project_id) .vpc_name(&default_name.clone().into()) .vpc_router_name(&system_name.clone().into()) - .router_route_name(&default_name.clone().into()) + .router_route_name(&"default-v4".parse::().unwrap().into()) + .fetch() + .await + .expect("Failed to fetch default route"); + datastore + .router_delete_route(&opctx, &authz_route) + .await + .expect("Failed to delete default route"); + + let (.., authz_route, _route) = LookupPath::new(&opctx, &datastore) + .project_id(project_id) + .vpc_name(&default_name.clone().into()) + .vpc_router_name(&system_name.clone().into()) + .router_route_name(&"default-v6".parse::().unwrap().into()) .fetch() .await .expect("Failed to fetch default route"); diff --git a/nexus/tests/integration_tests/router_routes.rs b/nexus/tests/integration_tests/router_routes.rs index 10c594bba9..a13026a7fd 100644 --- a/nexus/tests/integration_tests/router_routes.rs +++ b/nexus/tests/integration_tests/router_routes.rs @@ -10,6 +10,8 @@ use nexus_test_utils::identity_eq; use nexus_test_utils::resource_helpers::objects_list_page_authz; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; +use omicron_common::api::external::IpNet; +use omicron_common::api::external::SimpleIdentity; use omicron_common::api::external::{ IdentityMetadataCreateParams, IdentityMetadataUpdateParams, RouteDestination, RouteTarget, RouterRoute, RouterRouteKind, @@ -59,27 +61,48 @@ async fn test_router_routes(cptestctx: &ControlPlaneTestContext) { .await .items; - // The system should start with a single, pre-configured route - assert_eq!(system_router_routes.len(), 1); - - // That route should be the default route - let default_route = &system_router_routes[0]; - assert_eq!(default_route.kind, RouterRouteKind::Default); - - // It errors if you try to delete the default route - let error: dropshot::HttpErrorResponseBody = NexusRequest::expect_failure( - client, - StatusCode::BAD_REQUEST, - Method::DELETE, - get_route_url("system", "default").as_str(), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .unwrap() - .parsed_body() - .unwrap(); - assert_eq!(error.message, "DELETE not allowed on system routes"); + // The system should start with three preconfigured routes: + // - a default v4 gateway route + // - a default v6 gateway route + // - a managed subnet route for the 'default' subnet + assert_eq!(system_router_routes.len(), 3); + + let mut v4_route = None; + let mut v6_route = None; + let mut subnet_route = None; + for route in system_router_routes { + match (&route.kind, &route.destination, &route.target) { + (RouterRouteKind::Default, RouteDestination::IpNet(IpNet::V4(_)), RouteTarget::InternetGateway(_)) => {v4_route = Some(route);}, + (RouterRouteKind::Default, RouteDestination::IpNet(IpNet::V6(_)), RouteTarget::InternetGateway(_)) => {v6_route = Some(route);}, + (RouterRouteKind::VpcSubnet, RouteDestination::Subnet(n0), RouteTarget::Subnet(n1)) if n0 == n1 && n0.as_str() == "default" => {subnet_route = Some(route);}, + _ => panic!("unexpected system route {route:?} -- wanted gateway and subnet"), + } + } + + let v4_route = + v4_route.expect("no v4 gateway route found in system router"); + let v6_route = + v6_route.expect("no v6 gateway route found in system router"); + let subnet_route = + subnet_route.expect("no default subnet route found in system router"); + + // Deleting any default system route is disallowed. + for route in &[&v4_route, &v6_route, &subnet_route] { + let error: dropshot::HttpErrorResponseBody = + NexusRequest::expect_failure( + client, + StatusCode::BAD_REQUEST, + Method::DELETE, + get_route_url("system", route.name().as_str()).as_str(), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + assert_eq!(error.message, "DELETE not allowed on system routes"); + } // Create a custom router create_router(&client, project_name, vpc_name, router_name).await; From be9f8ab7be6795605c981c0b04e11187c600c197 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 16:46:00 +0100 Subject: [PATCH 20/39] Accidental local state... --- dev-tools/preprocessed_configs/config.xml | 41 ----------------------- 1 file changed, 41 deletions(-) delete mode 100644 dev-tools/preprocessed_configs/config.xml diff --git a/dev-tools/preprocessed_configs/config.xml b/dev-tools/preprocessed_configs/config.xml deleted file mode 100644 index 9b13f12aea..0000000000 --- a/dev-tools/preprocessed_configs/config.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - trace - true - - - 8123 - 9000 - 9004 - - ./ - - true - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - \ No newline at end of file From f433b38574e50b6759d132c9d27587c4e955b897 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 18:49:50 +0100 Subject: [PATCH 21/39] Unsubscribe routes from sled when ports are removed. --- dev-tools/omdb/tests/successes.out | 2 +- illumos-utils/src/opte/port.rs | 12 +++++ illumos-utils/src/opte/port_manager.rs | 72 ++++++++++++++++++-------- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 07960112d6..b1476b5f37 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -430,7 +430,7 @@ task: "metrics_producer_gc" currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms -warning: unknown background task: "metrics_producer_gc" (don't know how to interpret details: Object {"expiration": String(""), "pruned": Array []}) +warning: unknown background task: "metrics_producer_gc" (don't know how to interpret details: Object {"expiration": String(""), "pruned": Array []}) task: "phantom_disks" configured period: every 30s diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 38ba1b8c1c..832335891c 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -7,7 +7,9 @@ use crate::opte::Gateway; use crate::opte::Vni; use macaddr::MacAddr6; +use omicron_common::api::external; use omicron_common::api::external::IpNet; +use omicron_common::api::internal::shared::RouterId; use std::net::IpAddr; use std::sync::Arc; @@ -134,4 +136,14 @@ impl Port { pub fn slot(&self) -> u8 { self.inner.slot } + + pub fn system_router_key(&self) -> RouterId { + // Unwrap safety: both of these VNI types represent validated u24s. + let vni = external::Vni::try_from(self.vni().as_u32()).unwrap(); + RouterId { vni, subnet: None } + } + + pub fn custom_router_key(&self) -> RouterId { + RouterId { subnet: Some(*self.subnet()), ..self.system_router_key() } + } } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index d206273c48..943e818832 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -60,6 +60,7 @@ const XDE_LINK_PREFIX: &str = "opte"; struct RouteSet { version: Option, routes: HashSet, + active_ports: usize, } #[derive(Debug)] @@ -76,9 +77,6 @@ struct PortManagerInner { // (which includes the Uuid of the parent instance or service) ports: Mutex>, - // XX: Should this be the UUID of the VPC? The rulesets are - // arguably shared v4+v6, although today we don't yet - // allow dual-stack, let alone v6. // Map of all current resolved routes. routes: Mutex>, } @@ -380,12 +378,9 @@ impl PortManager { (port, ticket) }; - // XX: need to delete safely after all subnet-holders leave - // to not get flooded with useless rules. let mut routes = self.inner.routes.lock().unwrap(); - let system_routes = routes - .entry(RouterId { vni: nic.vni, subnet: None }) - .or_insert_with(|| { + let system_routes = + routes.entry(port.system_router_key()).or_insert_with(|| { let mut routes = HashSet::new(); // Services do not talk to one another via OPTE, but do need @@ -402,16 +397,20 @@ impl PortManager { }); } - RouteSet { version: None, routes } - }) - .clone(); + RouteSet { version: None, routes, active_ports: 0 } + }); + system_routes.active_ports += 1; + // Needed to get borrowck on our side, sadly. + let system_routes = system_routes.clone(); let custom_routes = routes - .entry(RouterId { vni: nic.vni, subnet: Some(nic.subnet) }) + .entry(port.custom_router_key()) .or_insert_with(|| RouteSet { version: None, routes: HashSet::default(), + active_ports: 0, }); + custom_routes.active_ports += 1; for (class, routes) in [ (RouterClass::System, &system_routes), @@ -460,11 +459,16 @@ impl PortManager { let mut routes = self.inner.routes.lock().unwrap(); let mut deltas = HashMap::new(); for set in new_routes { + // Disregard any route information for a subnet we don't have. + let Some(old) = routes.get(&set.id) else { + continue; + }; + // We have to handle subnet router changes, as well as // spurious updates from multiple Nexus instances. // If there's a UUID match, only update if vers increased, // otherwise take the update verbatim (including loss of version). - let (to_add, to_delete) = if let Some(old) = routes.get(&set.id) { + let (to_add, to_delete): (HashSet<_>, HashSet<_>) = match (old.version, set.version) { (Some(old_vers), Some(new_vers)) if !old_vers.is_replaced_by(&new_vers) => @@ -475,30 +479,29 @@ impl PortManager { set.routes.difference(&old.routes).cloned().collect(), old.routes.difference(&set.routes).cloned().collect(), ), - } - } else { - (set.routes.clone(), HashSet::new()) - }; + }; deltas.insert(set.id, (to_add, to_delete)); + let active_ports = old.active_ports; routes.insert( set.id, - RouteSet { version: set.version, routes: set.routes }, + RouteSet { + version: set.version, + routes: set.routes, + active_ports, + }, ); } - drop(routes); let ports = self.inner.ports.lock().unwrap(); #[cfg(target_os = "illumos")] let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?; for port in ports.values() { - let vni = external::Vni::try_from(port.vni().as_u32()).unwrap(); - let system_id = RouterId { vni, subnet: None }; + let system_id = port.system_router_key(); let system_delta = deltas.get(&system_id); - let custom_id = RouterId { vni, subnet: Some(*port.subnet()) }; - + let custom_id = port.custom_router_key(); let custom_delta = deltas.get(&custom_id); #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] @@ -821,6 +824,29 @@ impl PortTicket { ); return Err(Error::ReleaseMissingPort(self.id, self.kind)); }; + drop(ports); + + // Cleanup the set of subnets we want to receive routes for. + let mut routes = self.manager.routes.lock().unwrap(); + for key in [port.system_router_key(), port.custom_router_key()] { + let should_remove = routes + .get_mut(&key) + .map(|v| { + v.active_ports = v.active_ports.saturating_sub(1); + v.active_ports == 0 + }) + .unwrap_or_default(); + + if should_remove { + routes.remove(&key); + info!( + self.manager.log, + "Removed route set for subnet"; + "id" => ?&key, + ); + } + } + debug!( self.manager.log, "Removed OPTE port from manager"; From 62ca9f0bce8d053d4c6fb091672752e458612fbd Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 04:10:19 +0100 Subject: [PATCH 22/39] Migration query for subnet route creation. --- schema/crdb/vpc-subnet-routing/up03.sql | 108 ++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 schema/crdb/vpc-subnet-routing/up03.sql diff --git a/schema/crdb/vpc-subnet-routing/up03.sql b/schema/crdb/vpc-subnet-routing/up03.sql new file mode 100644 index 0000000000..9340de0c46 --- /dev/null +++ b/schema/crdb/vpc-subnet-routing/up03.sql @@ -0,0 +1,108 @@ +-- We need to manually rebuild a compliant set of routes. +-- Remove everything that exists today. +DELETE FROM omicron.public.router_route WHERE 1=1; + +-- Insert fixed_data routes for the services VPC. +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +VALUES + ( + '001de000-074c-4000-8000-000000000002', 'default-v4', + 'Default internet gateway route for Oxide Services', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'default', + 'inetgw:outbound', 'ipnet:0.0.0.0/0' + ), + ( + '001de000-074c-4000-8000-000000000003', 'default-v6', + 'Default internet gateway route for Oxide Services', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'default', + 'inetgw:outbound', 'ipnet:::/0' + ), + ( + '001de000-c470-4000-8000-000000000004', 'sn-external-dns', + 'Built-in VPC Subnet for Oxide service (external-dns)', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'vpc_subnet', + 'subnet:external-dns', 'subnet:external-dns' + ), + ( + '001de000-c470-4000-8000-000000000005', 'sn-nexus', + 'Built-in VPC Subnet for Oxide service (nexus)', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'vpc_subnet', + 'subnet:nexus', 'subnet:nexus' + ), + ( + '001de000-c470-4000-8000-000000000006', 'sn-boundary-ntp', + 'Built-in VPC Subnet for Oxide service (nexus)', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'vpc_subnet', + 'subnet:boundary-ntp', 'subnet:boundary-ntp' + ) +ON CONFLICT DO NOTHING; + +-- Insert gateway routes for user VPCs. +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +SELECT + gen_random_uuid(), 'default-v4', + 'The default route of a vpc', + now(), now(), + omicron.public.vpc_router.id, 'default', + 'inetgw:outbound', 'ipnet:0.0.0.0/0' +FROM + omicron.public.vpc_router +ON CONFLICT DO NOTHING; + +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +SELECT + gen_random_uuid(), 'default-v6', + 'The default route of a vpc', + now(), now(), + omicron.public.vpc_router.id, 'default', + 'inetgw:outbound', 'ipnet:::/0' +FROM + omicron.public.vpc_router +ON CONFLICT DO NOTHING; + +-- Insert subnet routes for every defined VPC subnet. +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +SELECT + gen_random_uuid(), 'sn-' || vpc_subnet.name, + 'VPC Subnet route for ''' || vpc_subnet.name || '''', + now(), now(), + omicron.public.vpc_router.id, 'default', + 'subnet:' || vpc_subnet.name, 'subnet:' || vpc_subnet.name +FROM + (omicron.public.vpc_subnet JOIN omicron.public.vpc + ON vpc_subnet.vpc_id = vpc.id) JOIN omicron.public.vpc_router + ON vpc_router.vpc_id = vpc.id +ON CONFLICT DO NOTHING; From 2c06ff48b4684a96bf41073f7ef3f4ba42cff625 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 11:12:06 +0100 Subject: [PATCH 23/39] Rework migration slightly. --- schema/crdb/vpc-subnet-routing/up03.sql | 84 +++++++++++-------------- 1 file changed, 36 insertions(+), 48 deletions(-) diff --git a/schema/crdb/vpc-subnet-routing/up03.sql b/schema/crdb/vpc-subnet-routing/up03.sql index 9340de0c46..d256921d34 100644 --- a/schema/crdb/vpc-subnet-routing/up03.sql +++ b/schema/crdb/vpc-subnet-routing/up03.sql @@ -1,55 +1,10 @@ +set local disallow_full_table_scans = off; + -- We need to manually rebuild a compliant set of routes. -- Remove everything that exists today. DELETE FROM omicron.public.router_route WHERE 1=1; --- Insert fixed_data routes for the services VPC. -INSERT INTO omicron.public.router_route - ( - id, name, - description, - time_created, time_modified, - vpc_router_id, kind, - target, destination - ) -VALUES - ( - '001de000-074c-4000-8000-000000000002', 'default-v4', - 'Default internet gateway route for Oxide Services', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'default', - 'inetgw:outbound', 'ipnet:0.0.0.0/0' - ), - ( - '001de000-074c-4000-8000-000000000003', 'default-v6', - 'Default internet gateway route for Oxide Services', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'default', - 'inetgw:outbound', 'ipnet:::/0' - ), - ( - '001de000-c470-4000-8000-000000000004', 'sn-external-dns', - 'Built-in VPC Subnet for Oxide service (external-dns)', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'vpc_subnet', - 'subnet:external-dns', 'subnet:external-dns' - ), - ( - '001de000-c470-4000-8000-000000000005', 'sn-nexus', - 'Built-in VPC Subnet for Oxide service (nexus)', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'vpc_subnet', - 'subnet:nexus', 'subnet:nexus' - ), - ( - '001de000-c470-4000-8000-000000000006', 'sn-boundary-ntp', - 'Built-in VPC Subnet for Oxide service (nexus)', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'vpc_subnet', - 'subnet:boundary-ntp', 'subnet:boundary-ntp' - ) -ON CONFLICT DO NOTHING; - --- Insert gateway routes for user VPCs. +-- Insert gateway routes for all VPCs. INSERT INTO omicron.public.router_route ( id, name, @@ -106,3 +61,36 @@ FROM ON vpc_subnet.vpc_id = vpc.id) JOIN omicron.public.vpc_router ON vpc_router.vpc_id = vpc.id ON CONFLICT DO NOTHING; + +-- Replace IDs of fixed_data routes for the services VPC. +-- This is done instead of an insert to match the initial +-- empty state of dbinit.sql. +WITH known_ids (new_id, new_name, new_description) AS ( + VALUES + ( + '001de000-074c-4000-8000-000000000002', 'default-v4', + 'Default internet gateway route for Oxide Services' + ), + ( + '001de000-074c-4000-8000-000000000003', 'default-v6', + 'Default internet gateway route for Oxide Services' + ), + ( + '001de000-c470-4000-8000-000000000004', 'sn-external-dns', + 'Built-in VPC Subnet for Oxide service (external-dns)' + ), + ( + '001de000-c470-4000-8000-000000000005', 'sn-nexus', + 'Built-in VPC Subnet for Oxide service (nexus)' + ), + ( + '001de000-c470-4000-8000-000000000006', 'sn-boundary-ntp', + 'Built-in VPC Subnet for Oxide service (boundary-ntp)' + ) +) +UPDATE omicron.public.router_route +SET + id = CAST(new_id AS UUID), + description = new_description +FROM known_ids +WHERE vpc_router_id = '001de000-074c-4000-8000-000000000001' AND new_name = router_route.name; From 0e8d1adf57f3041436deed9c5829e1a58068b0be Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 11:22:48 +0100 Subject: [PATCH 24/39] Bump OPTE to include latest perf work. --- Cargo.lock | 12 ++++++------ Cargo.toml | 4 ++-- tools/opte_version | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad3ac965d0..924cf4d0f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3470,7 +3470,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" [[package]] name = "illumos-utils" @@ -3883,7 +3883,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "quote", "syn 2.0.64", @@ -6008,7 +6008,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "cfg-if", "dyn-clone", @@ -6025,7 +6025,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -6037,7 +6037,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -6111,7 +6111,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 8d298d1feb..f12c6c72f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -347,14 +347,14 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "d6177ca84f23e60a661461bb4cece475689502d2", features = [ "api", "std" ] } once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "d6177ca84f23e60a661461bb4cece475689502d2" } oso = "0.27" owo-colors = "4.0.0" oximeter = { path = "oximeter/oximeter" } diff --git a/tools/opte_version b/tools/opte_version index fc3e603c41..6126a52eb4 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.31.258 +0.31.259 From 25372224b295cbd4175252876de9809aef35a479 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 14:26:14 +0100 Subject: [PATCH 25/39] Self-review pt.1. --- clients/sled-agent-client/src/lib.rs | 5 +- common/src/api/internal/shared.rs | 27 ++++--- illumos-utils/src/opte/mod.rs | 2 + illumos-utils/src/opte/port_manager.rs | 57 +++++++++------ nexus/db-model/src/schema_versions.rs | 2 +- nexus/db-queries/src/db/datastore/instance.rs | 52 ------------- .../src/db/datastore/network_interface.rs | 4 +- nexus/db-queries/src/db/datastore/vpc.rs | 73 ++++++++++++++++--- nexus/db-queries/src/db/fixed_data/vpc.rs | 3 +- .../src/db/fixed_data/vpc_subnet.rs | 3 + nexus/src/app/background/vpc_routes.rs | 8 +- nexus/src/app/sagas/vpc_create.rs | 2 - nexus/src/app/vpc_router.rs | 2 - openapi/sled-agent.json | 38 +++++----- schema/crdb/dbinit.sql | 2 +- sled-agent/src/http_entrypoints.rs | 6 +- sled-agent/src/probe_manager.rs | 2 +- sled-agent/src/sled_agent.rs | 7 +- 18 files changed, 156 insertions(+), 139 deletions(-) diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index aaa59d0e98..4910918884 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -35,7 +35,6 @@ progenitor::generate_api!( PortConfigV1 = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, RouteConfig = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, IpNet = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, - RouterId = { derives = [PartialEq, Eq, Hash, Debug, Deserialize, Serialize] }, VirtualNetworkInterfaceHost = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] }, }, @@ -54,8 +53,8 @@ progenitor::generate_api!( PortFec = omicron_common::api::internal::shared::PortFec, PortSpeed = omicron_common::api::internal::shared::PortSpeed, RouterId = omicron_common::api::internal::shared::RouterId, - ReifiedVpcRoute = omicron_common::api::internal::shared::ReifiedVpcRoute, - ReifiedVpcRouteSet = omicron_common::api::internal::shared::ReifiedVpcRouteSet, + ResolvedVpcRoute = omicron_common::api::internal::shared::ResolvedVpcRoute, + ResolvedVpcRouteSet = omicron_common::api::internal::shared::ResolvedVpcRouteSet, RouterTarget = omicron_common::api::internal::shared::RouterTarget, RouterVersion = omicron_common::api::internal::shared::RouterVersion, SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig, diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 52601a01a8..51b829e214 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -594,7 +594,7 @@ impl TryFrom<&[IpNetwork]> for IpAllowList { #[derive( Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] -pub struct ReifiedVpcRoute { +pub struct ResolvedVpcRoute { pub dest: IpNet, pub target: RouterTarget, } @@ -611,23 +611,28 @@ pub enum RouterTarget { VpcSubnet(IpNet), } -/// XXX +/// Information on the current parent router (and version) of a route set +/// according to the control plane. #[derive( Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] pub struct RouterVersion { pub router_id: Uuid, - pub generation: u64, + pub version: u64, } impl RouterVersion { + /// Return whether a new route set should be applied over the current + /// values. + /// + /// This will occur when seeing a new version and a matching parent, + /// or a new parent router on the control plane. pub fn is_replaced_by(&self, other: &Self) -> bool { - (self.router_id != other.router_id) - || self.generation < other.generation + (self.router_id != other.router_id) || self.version < other.version } } -/// Implementation details on XXX +/// Identifier for a VPC and/or subnet. #[derive( Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] @@ -636,19 +641,19 @@ pub struct RouterId { pub subnet: Option, } -/// Version information +/// Version information for routes on a given VPC subnet. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] -pub struct ReifiedVpcRouteState { +pub struct ResolvedVpcRouteState { pub id: RouterId, pub version: Option, } -/// An updated set of routes for a given +/// An updated set of routes for a given VPC and/or subnet. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] -pub struct ReifiedVpcRouteSet { +pub struct ResolvedVpcRouteSet { pub id: RouterId, pub version: Option, - pub routes: HashSet, + pub routes: HashSet, } #[cfg(test)] diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index f6ef186808..e53468f40e 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -73,6 +73,7 @@ impl Gateway { } } +/// Convert a nexus `IpNet` to an OPTE `IpCidr`. fn net_to_cidr(net: IpNet) -> IpCidr { match net { IpNet::V4(net) => IpCidr::Ip4(Ipv4Cidr::new( @@ -86,6 +87,7 @@ fn net_to_cidr(net: IpNet) -> IpCidr { } } +/// Convert a nexus `RouterTarget` to an OPTE `RouterTarget`. fn router_target_opte(target: &shared::RouterTarget) -> RouterTarget { use shared::RouterTarget::*; match target { diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 769634742b..caeda81217 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -16,9 +16,9 @@ use ipnetwork::IpNetwork; use omicron_common::api::external; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; -use omicron_common::api::internal::shared::ReifiedVpcRoute; -use omicron_common::api::internal::shared::ReifiedVpcRouteSet; -use omicron_common::api::internal::shared::ReifiedVpcRouteState; +use omicron_common::api::internal::shared::ResolvedVpcRoute; +use omicron_common::api::internal::shared::ResolvedVpcRouteSet; +use omicron_common::api::internal::shared::ResolvedVpcRouteState; use omicron_common::api::internal::shared::RouterId; use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget; use omicron_common::api::internal::shared::RouterVersion; @@ -55,10 +55,11 @@ use uuid::Uuid; // Prefix used to identify xde data links. const XDE_LINK_PREFIX: &str = "opte"; +/// Stored routes (and usage count) for a given VPC/subnet. #[derive(Debug, Clone)] struct RouteSet { version: Option, - routes: HashSet, + routes: HashSet, active_ports: usize, } @@ -66,17 +67,17 @@ struct RouteSet { struct PortManagerInner { log: Logger, - // Sequential identifier for each port on the system. + /// Sequential identifier for each port on the system. next_port_id: AtomicU64, - // IP address of the hosting sled on the underlay. + /// IP address of the hosting sled on the underlay. underlay_ip: Ipv6Addr, - // Map of all ports, keyed on the interface Uuid and its kind - // (which includes the Uuid of the parent instance or service) + /// Map of all ports, keyed on the interface Uuid and its kind + /// (which includes the Uuid of the parent instance or service) ports: Mutex>, - // Map of all current resolved routes. + /// Map of all current resolved routes. routes: Mutex>, } @@ -377,6 +378,10 @@ impl PortManager { (port, ticket) }; + // Check locally to see whether we have any routes from the + // control plane for this port already installed. If not, + // create a record to show that we're interested in receiving + // those routes. let mut routes = self.inner.routes.lock().unwrap(); let system_routes = routes.entry(port.system_router_key()).or_insert_with(|| { @@ -386,11 +391,11 @@ impl PortManager { // to reach out over the Internet *before* nexus is up to give // us real rules. The easiest bet is to instantiate these here. if is_service { - routes.insert(ReifiedVpcRoute { + routes.insert(ResolvedVpcRoute { dest: "0.0.0.0/0".parse().unwrap(), target: ApiRouterTarget::InternetGateway, }); - routes.insert(ReifiedVpcRoute { + routes.insert(ResolvedVpcRoute { dest: "::/0".parse().unwrap(), target: ApiRouterTarget::InternetGateway, }); @@ -399,7 +404,7 @@ impl PortManager { RouteSet { version: None, routes, active_ports: 0 } }); system_routes.active_ports += 1; - // Needed to get borrowck on our side, sadly. + // Clone is needed to get borrowck on our side, sadly. let system_routes = system_routes.clone(); let custom_routes = routes @@ -443,23 +448,23 @@ impl PortManager { Ok((port, ticket)) } - pub fn vpc_routes_list(&self) -> Vec { + pub fn vpc_routes_list(&self) -> Vec { let routes = self.inner.routes.lock().unwrap(); routes .iter() - .map(|(k, v)| ReifiedVpcRouteState { id: *k, version: v.version }) + .map(|(k, v)| ResolvedVpcRouteState { id: *k, version: v.version }) .collect() } pub fn vpc_routes_ensure( &self, - new_routes: Vec, + new_routes: Vec, ) -> Result<(), Error> { let mut routes = self.inner.routes.lock().unwrap(); let mut deltas = HashMap::new(); - for set in new_routes { + for new in new_routes { // Disregard any route information for a subnet we don't have. - let Some(old) = routes.get(&set.id) else { + let Some(old) = routes.get(&new.id) else { continue; }; @@ -468,34 +473,38 @@ impl PortManager { // If there's a UUID match, only update if vers increased, // otherwise take the update verbatim (including loss of version). let (to_add, to_delete): (HashSet<_>, HashSet<_>) = - match (old.version, set.version) { + match (old.version, new.version) { (Some(old_vers), Some(new_vers)) if !old_vers.is_replaced_by(&new_vers) => { continue; } _ => ( - set.routes.difference(&old.routes).cloned().collect(), - old.routes.difference(&set.routes).cloned().collect(), + new.routes.difference(&old.routes).cloned().collect(), + old.routes.difference(&new.routes).cloned().collect(), ), }; - deltas.insert(set.id, (to_add, to_delete)); + deltas.insert(new.id, (to_add, to_delete)); let active_ports = old.active_ports; routes.insert( - set.id, + new.id, RouteSet { - version: set.version, - routes: set.routes, + version: new.version, + routes: new.routes, active_ports, }, ); } + // Note: We're deliberately holding both locks here + // to prevent several nexuses computng and applying deltas + // out of order. let ports = self.inner.ports.lock().unwrap(); #[cfg(target_os = "illumos")] let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?; + // Propagate deltas out to all ports. for port in ports.values() { let system_id = port.system_router_key(); let system_delta = deltas.get(&system_id); diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index e66c89f86f..db783af78b 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(64, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(65, 0, 0); /// List of all past database schema versions, in *reverse* order /// diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index cd12cb6793..ce40e20501 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -441,58 +441,6 @@ impl DataStore { Ok(result) } - /// Lists all instances on in-service sleds with active Propolis VMM - /// processes, returning the instance along with the VMM on which it's - /// running, the sled on which the VMM is running, and the project that owns - /// the instance. - /// - /// The query performed by this function is paginated by the sled's UUID. - pub async fn instance_and_vpc_list_by_sled_agent( - &self, - opctx: &OpContext, - pagparams: &DataPageParams<'_, Uuid>, - ) -> ListResultVec<(Sled, Instance, Vmm, Project)> { - use crate::db::schema::{ - instance::dsl as instance_dsl, project::dsl as project_dsl, - sled::dsl as sled_dsl, vmm::dsl as vmm_dsl, - }; - opctx.authorize(authz::Action::Read, &authz::FLEET).await?; - let conn = self.pool_connection_authorized(opctx).await?; - - let result = paginated(sled_dsl::sled, sled_dsl::id, pagparams) - .filter(sled_dsl::time_deleted.is_null()) - .sled_filter(SledFilter::InService) - .inner_join( - vmm_dsl::vmm - .on(vmm_dsl::sled_id - .eq(sled_dsl::id) - .and(vmm_dsl::time_deleted.is_null())) - .inner_join( - instance_dsl::instance - .on(instance_dsl::id - .eq(vmm_dsl::instance_id) - .and(instance_dsl::time_deleted.is_null())) - .inner_join( - project_dsl::project.on(project_dsl::id - .eq(instance_dsl::project_id) - .and(project_dsl::time_deleted.is_null())), - ), - ), - ) - .sled_filter(SledFilter::InService) - .select(( - Sled::as_select(), - Instance::as_select(), - Vmm::as_select(), - Project::as_select(), - )) - .load_async::<(Sled, Instance, Vmm, Project)>(&*conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; - - Ok(result) - } - pub async fn project_delete_instance( &self, opctx: &OpContext, diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 6baa6f643f..c8e071684b 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -149,8 +149,8 @@ impl DataStore { // NIC of that instance. Accordingly, NIC create may cause dangling // entries to re-resolve to a valid instance (even if it is not yet // started). - // This will not trigger the RPW directly, we still need to do so - // in e.g. the instance watcher task. + // This will not trigger the route RPW directly, we still need to do + // so in e.g. the instance watcher task. if out.primary { self.vpc_increment_rpw_version(opctx, out.vpc_id) .await diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index fd90c8a7e6..acc6e278d3 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -66,7 +66,7 @@ use omicron_common::api::external::RouteTarget; use omicron_common::api::external::RouterRouteKind as ExternalRouteKind; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::Vni as ExternalVni; -use omicron_common::api::internal::shared::ReifiedVpcRoute; +use omicron_common::api::internal::shared::ResolvedVpcRoute; use omicron_common::api::internal::shared::RouterTarget; use ref_cast::RefCast; use std::collections::BTreeMap; @@ -1064,6 +1064,17 @@ impl DataStore { .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + // Unlink all subnets from this router. + // XXX: We might this want to error out before the delete fires. + use db::schema::vpc_subnet::dsl as vpc; + diesel::update(vpc::vpc_subnet) + .filter(vpc::time_deleted.is_null()) + .filter(vpc::custom_router_id.eq(authz_router.id())) + .set(vpc::custom_router_id.eq(Option::::None)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) } @@ -1312,7 +1323,8 @@ impl DataStore { // aren't something which they can meaningfully interact with, // so uuid stability on e.g. VPC rename is not a primary concern. // We make sure only to alter VPC subnet rules here: users may - // modify other system routes like internet gateways. + // modify other system routes like internet gateways (which are + // `RouteKind::Default`). let conn = self.pool_connection_authorized(opctx).await?; let log = opctx.log.clone(); self.transaction_retry_wrapper("vpc_subnet_route_reconcile") @@ -1364,7 +1376,7 @@ impl DataStore { } } - // Add/Remove routes. Retry if numebr is incorrect due to + // Add/Remove routes. Retry if number is incorrect due to // concurrent modification. let now = Utc::now(); let to_update = invalid.len(); @@ -1407,6 +1419,27 @@ impl DataStore { } } + // Verify that route set is exactly as intended, and rollback otherwise. + let current_rules: Vec = dsl::router_route + .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::vpc_router_id.eq(system_router_id)) + .select(RouterRoute::as_select()) + .load_async(&conn) + .await?; + + if current_rules.len() != expected_names.len() { + return Err(DieselError::RollbackTransaction) + } + + for rule in current_rules { + match (rule.kind.0, rule.target.0) { + (ExternalRouteKind::VpcSubnet, RouteTarget::Subnet(n)) + if expected_names.contains(Name::ref_cast(&n)) => {}, + _ => return Err(DieselError::RollbackTransaction), + } + } + Ok(()) }}).await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; @@ -1420,7 +1453,6 @@ impl DataStore { opctx: &OpContext, vpc_id: Uuid, ) -> LookupResult { - // use db::schema::vpc::dsl as vpc_dsl; use db::schema::vpc::dsl as vpc_dsl; use db::schema::vpc_router::dsl as router_dsl; @@ -1486,7 +1518,7 @@ impl DataStore { &self, opctx: &OpContext, vpc_router_id: Uuid, - ) -> Result, Error> { + ) -> Result, Error> { // Get all rules in target router. opctx.check_complex_operations_allowed()?; @@ -1511,7 +1543,12 @@ impl DataStore { all_rules.extend(batch); } - // XXX: transaction based on generation number? + // This is not in a transaction, because... + // We're not necessarily too concerned about getting partially + // updated state when resolving these names. See the header discussion + // in `nexus/src/app/background/vpc_routes.rs`: any state updates + // are followed by a version bump/notify, so we will be eventually + // consistent with route resolution. let mut subnet_names = HashSet::new(); let mut vpc_names = HashSet::new(); let mut inetgw_names = HashSet::new(); @@ -1544,7 +1581,7 @@ impl DataStore { } } - // TODO: transact these, and/or solve in fewer queries. + // TODO: This would be nice to solve in fewer queries. let mut subnets = HashMap::new(); for name in subnet_names.drain() { if let Ok((.., subnet)) = db::lookup::LookupPath::new(opctx, self) @@ -1576,7 +1613,7 @@ impl DataStore { .fetch() .await { - // XXX: currently an instance can have one primary, + // XXX: currently an instance can have one primary NIC, // and it is not dual-stack (v4 + v6). We need // to clarify what should be resolved in the v6 case. if let Ok(primary_nic) = self @@ -1676,11 +1713,11 @@ impl DataStore { }; if let (Some(dest), Some(target)) = (v4_dest, v4_target) { - out.insert(ReifiedVpcRoute { dest, target }); + out.insert(ResolvedVpcRoute { dest, target }); } if let (Some(dest), Some(target)) = (v6_dest, v6_target) { - out.insert(ReifiedVpcRoute { dest, target }); + out.insert(ResolvedVpcRoute { dest, target }); } } @@ -2507,4 +2544,20 @@ mod tests { routes } + + // Test to verify that VPC routers resolve to the primary addr + // of an instance NIC. + #[tokio::test] + async fn test_vpc_router_rule_instance_resolve() { + // use vpc_resolve_router_rules. + todo!() + } + + // Test to verify that VPC routers resolve rules intelligently + // across dual IPv4 / IPv6 targets/destinations. + #[tokio::test] + async fn test_vpc_router_rule_v4_v6_resolve() { + // use vpc_resolve_router_rules. + todo!() + } } diff --git a/nexus/db-queries/src/db/fixed_data/vpc.rs b/nexus/db-queries/src/db/fixed_data/vpc.rs index 6dffc11426..604e939680 100644 --- a/nexus/db-queries/src/db/fixed_data/vpc.rs +++ b/nexus/db-queries/src/db/fixed_data/vpc.rs @@ -23,7 +23,7 @@ pub static SERVICES_VPC_ROUTER_ID: Lazy = Lazy::new(|| { .expect("invalid uuid for builtin services vpc router id") }); -/// UUID of default route for built-in Services VPC. +/// UUID of default IPv4 route for built-in Services VPC. pub static SERVICES_VPC_DEFAULT_V4_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-074c-4000-8000-000000000002" @@ -31,6 +31,7 @@ pub static SERVICES_VPC_DEFAULT_V4_ROUTE_ID: Lazy = .expect("invalid uuid for builtin services vpc default route id") }); +/// UUID of default IPv6 route for built-in Services VPC. pub static SERVICES_VPC_DEFAULT_V6_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-074c-4000-8000-000000000003" diff --git a/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs b/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs index 45db9b7e0b..7b2cc468ab 100644 --- a/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs +++ b/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs @@ -31,18 +31,21 @@ pub static NTP_VPC_SUBNET_ID: Lazy = Lazy::new(|| { .expect("invalid uuid for builtin boundary ntp vpc subnet id") }); +/// UUID of built-in subnet route VPC Subnet route for External DNS. pub static DNS_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-c470-4000-8000-000000000004" .parse() .expect("invalid uuid for builtin services vpc default route id") }); +/// UUID of built-in subnet route VPC Subnet route for Nexus. pub static NEXUS_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-c470-4000-8000-000000000005" .parse() .expect("invalid uuid for builtin services vpc default route id") }); +/// UUID of built-in subnet route VPC Subnet route for Boundary NTP. pub static NTP_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-c470-4000-8000-000000000006" .parse() diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index 6cdf720ce2..359b86f939 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -15,7 +15,7 @@ use nexus_types::{ identity::Resource, }; use omicron_common::api::internal::shared::{ - ReifiedVpcRoute, ReifiedVpcRouteSet, RouterId, RouterVersion, + ResolvedVpcRoute, ResolvedVpcRouteSet, RouterId, RouterVersion, }; use serde_json::json; use std::collections::hash_map::Entry; @@ -78,7 +78,7 @@ impl BackgroundTask for VpcRouteManager { }) .collect(); - let mut known_rules: HashMap> = + let mut known_rules: HashMap> = HashMap::new(); let mut db_routers = HashMap::new(); let mut vni_to_vpc = HashMap::new(); @@ -184,7 +184,7 @@ impl BackgroundTask for VpcRouteManager { let mut to_push = Vec::new(); let mut set_rules = |id, version, routes| { - to_push.push(ReifiedVpcRouteSet { id, routes, version }); + to_push.push(ResolvedVpcRouteSet { id, routes, version }); }; // resolve into known_rules on an as-needed basis. @@ -199,7 +199,7 @@ impl BackgroundTask for VpcRouteManager { let router_id = db_router.id(); let version = RouterVersion { - generation: db_router.resolved_version as u64, + version: db_router.resolved_version as u64, router_id, }; diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index c84cf5ff20..22db026c4f 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -228,8 +228,6 @@ async fn svc_create_router_undo( Ok(()) } -// XX: possibly do these as a subsaga? - async fn svc_create_v4_route( sagactx: NexusActionContext, ) -> Result<(), ActionError> { diff --git a/nexus/src/app/vpc_router.rs b/nexus/src/app/vpc_router.rs index ae2fdffeeb..40b4c1de0f 100644 --- a/nexus/src/app/vpc_router.rs +++ b/nexus/src/app/vpc_router.rs @@ -118,8 +118,6 @@ impl super::Nexus { .await } - // TODO(now): When a router is deleted it should be unassociated w/ any subnets it may be associated with - // or trigger an error. pub(crate) async fn vpc_delete_router( &self, opctx: &OpContext, diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index d332a9de01..5480f0bcf1 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -967,10 +967,10 @@ "content": { "application/json": { "schema": { - "title": "Array_of_ReifiedVpcRouteState", + "title": "Array_of_ResolvedVpcRouteState", "type": "array", "items": { - "$ref": "#/components/schemas/ReifiedVpcRouteState" + "$ref": "#/components/schemas/ResolvedVpcRouteState" } } } @@ -991,10 +991,10 @@ "content": { "application/json": { "schema": { - "title": "Array_of_ReifiedVpcRouteSet", + "title": "Array_of_ResolvedVpcRouteSet", "type": "array", "items": { - "$ref": "#/components/schemas/ReifiedVpcRouteSet" + "$ref": "#/components/schemas/ResolvedVpcRouteSet" } } } @@ -4290,7 +4290,7 @@ "rack_subnet" ] }, - "ReifiedVpcRoute": { + "ResolvedVpcRoute": { "description": "A VPC route resolved into a concrete target.", "type": "object", "properties": { @@ -4306,8 +4306,8 @@ "target" ] }, - "ReifiedVpcRouteSet": { - "description": "An updated set of routes for a given", + "ResolvedVpcRouteSet": { + "description": "An updated set of routes for a given VPC and/or subnet.", "type": "object", "properties": { "id": { @@ -4316,7 +4316,7 @@ "routes": { "type": "array", "items": { - "$ref": "#/components/schemas/ReifiedVpcRoute" + "$ref": "#/components/schemas/ResolvedVpcRoute" }, "uniqueItems": true }, @@ -4334,8 +4334,8 @@ "routes" ] }, - "ReifiedVpcRouteState": { - "description": "Version information", + "ResolvedVpcRouteState": { + "description": "Version information for routes on a given VPC subnet.", "type": "object", "properties": { "id": { @@ -4384,7 +4384,7 @@ ] }, "RouterId": { - "description": "Implementation details on XXX", + "description": "Identifier for a VPC and/or subnet.", "type": "object", "properties": { "subnet": { @@ -4474,22 +4474,22 @@ ] }, "RouterVersion": { - "description": "XXX", + "description": "Information on the current parent router (and version) of a route set according to the control plane.", "type": "object", "properties": { - "generation": { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, "router_id": { "type": "string", "format": "uuid" + }, + "version": { + "type": "integer", + "format": "uint64", + "minimum": 0 } }, "required": [ - "generation", - "router_id" + "router_id", + "version" ] }, "SemverVersion": { diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index fc70d10f3d..f354ea2fec 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3935,7 +3935,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '64.0.0', NULL) + (TRUE, NOW(), NOW(), '65.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index ff5ab393ea..3ec905960c 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -32,7 +32,7 @@ use omicron_common::api::internal::nexus::{ DiskRuntimeState, SledInstanceState, UpdateArtifactId, }; use omicron_common::api::internal::shared::{ - ReifiedVpcRouteSet, ReifiedVpcRouteState, SwitchPorts, + ResolvedVpcRouteSet, ResolvedVpcRouteState, SwitchPorts, }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -1036,7 +1036,7 @@ async fn bootstore_status( }] async fn list_vpc_routes( request_context: RequestContext, -) -> Result>, HttpError> { +) -> Result>, HttpError> { let sa = request_context.context(); Ok(HttpResponseOk(sa.list_vpc_routes())) } @@ -1048,7 +1048,7 @@ async fn list_vpc_routes( }] async fn set_vpc_routes( request_context: RequestContext, - body: TypedBody>, + body: TypedBody>, ) -> Result { let sa = request_context.context(); sa.set_vpc_routes(body.into_inner())?; diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index eabf3850af..40af604645 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -185,7 +185,7 @@ impl ProbeManagerInner { self.remove(current.difference(&target)).await; self.check(current.intersection(&target)).await; - // If we have created some new probes, we may (in future) need the control plane + // If we have created some new probes, we may need the control plane // to provide us with valid routes for the VPC the probe belongs to. if n_added > 0 { if let Err(e) = self diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index ee33733718..facbd9db88 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -49,7 +49,8 @@ use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; use omicron_common::api::internal::shared::{ - HostPortConfig, RackNetworkConfig, ReifiedVpcRouteSet, ReifiedVpcRouteState, + HostPortConfig, RackNetworkConfig, ResolvedVpcRouteSet, + ResolvedVpcRouteState, }; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, @@ -1095,13 +1096,13 @@ impl SledAgent { self.inner.bootstore.clone() } - pub fn list_vpc_routes(&self) -> Vec { + pub fn list_vpc_routes(&self) -> Vec { self.inner.port_manager.vpc_routes_list() } pub fn set_vpc_routes( &self, - routes: Vec, + routes: Vec, ) -> Result<(), Error> { self.inner.port_manager.vpc_routes_ensure(routes).map_err(Error::from) } From f217bd19a3a55f28b25f59c24a4a1c93de662aae Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 14:39:17 +0100 Subject: [PATCH 26/39] Self-review pt.2. --- nexus/src/app/background/vpc_routes.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index 359b86f939..f9b37d2175 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -35,10 +35,20 @@ impl VpcRouteManager { } } -// There's a sort of eventual consistency happening here. -// ... DETAIL XX ... -// version bumps must happen AFTER other changes occur in -// children etc. to keep this sane and working. :) +// This RPW doesn't concern itself overly much with resolved router targets +// and destinations being partial wrt. the current generation, in the same +// vein as how firewall rules are handled. Gating *pushing* this update on a +// generation number can be a bit more risky, but there's a sort of eventual +// consistency happening here that keeps this safe. +// +// Any location which updates name-resolvable state follows the pattern: +// * Update state. +// * Update (VPC-wide) router generation numbers. +// * Awaken this task. This might happen indirectly via e.g. instance start. +// +// As a result, any update which accidentally sees partial state will be followed +// by re-triggering this RPW with a higher generation number, giving us a re-resolved +// route set and pushing to any relevant sleds. impl BackgroundTask for VpcRouteManager { fn activate<'a>( &'a mut self, @@ -47,7 +57,6 @@ impl BackgroundTask for VpcRouteManager { async { let log = &opctx.log; - // XX: copied from omicron#5566 let sleds = match self .datastore .sled_list_all_batched(opctx, SledFilter::InService) @@ -99,7 +108,7 @@ impl BackgroundTask for VpcRouteManager { // based on the set of VNIs reported by this sled. // These provide the versions we'll stick with -- in the worst // case we push newer state to a sled with an older generation - // number, which + // number, which will be fixed up on the next activation. for set in &route_sets { let db_vni = Vni(set.id.vni); let maybe_vpc = vni_to_vpc.entry(set.id.vni); From b549044dade886b4218abf45a04394ee4a6ec5b9 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 22:42:37 +0100 Subject: [PATCH 27/39] Test route resolution. --- nexus/db-queries/src/db/datastore/silo.rs | 2 +- nexus/db-queries/src/db/datastore/vpc.rs | 325 +++++++++++++++++----- nexus/src/app/background/vpc_routes.rs | 11 +- 3 files changed, 262 insertions(+), 76 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs index 0fd858b900..76efcf99b0 100644 --- a/nexus/db-queries/src/db/datastore/silo.rs +++ b/nexus/db-queries/src/db/datastore/silo.rs @@ -106,7 +106,7 @@ impl DataStore { Ok(()) } - async fn silo_create_query( + pub(crate) async fn silo_create_query( opctx: &OpContext, silo: Silo, ) -> Result, Error> { diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index acc6e278d3..a5ceb9a2c0 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -66,7 +66,6 @@ use omicron_common::api::external::RouteTarget; use omicron_common::api::external::RouterRouteKind as ExternalRouteKind; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::Vni as ExternalVni; -use omicron_common::api::internal::shared::ResolvedVpcRoute; use omicron_common::api::internal::shared::RouterTarget; use ref_cast::RefCast; use std::collections::BTreeMap; @@ -1518,16 +1517,16 @@ impl DataStore { &self, opctx: &OpContext, vpc_router_id: Uuid, - ) -> Result, Error> { + ) -> Result, Error> { // Get all rules in target router. opctx.check_complex_operations_allowed()?; let (.., authz_project, authz_vpc, authz_router) = db::lookup::LookupPath::new(opctx, self) .vpc_router_id(vpc_router_id) - .lookup_for(authz::Action::ListChildren) + .lookup_for(authz::Action::Read) .await - .internal_context("lookup built-in services project")?; + .internal_context("lookup router by id for rules")?; let mut paginator = Paginator::new(SQL_BATCH_SIZE); let mut all_rules = vec![]; while let Some(p) = paginator.next() { @@ -1633,7 +1632,7 @@ impl DataStore { // how we should resolve name misses in route resolution. // This method adopts the same strategy: a lookup failure corresponds // to a NO-OP rule. - let mut out = HashSet::new(); + let mut out = HashMap::new(); for rule in all_rules { // Some dests/targets (e.g., subnet) resolve to *several* specifiers // to handle both v4 and v6. The user-facing API will prevent severe @@ -1712,12 +1711,18 @@ impl DataStore { RouteTarget::Vpc(_) => (None, None), }; + // XXX: Is there another way we should be handling destination + // collisions within a router? 'first/last wins' is fairly + // arbitrary when lookups are sorted on UUID, but it's + // unpredictable. + // It would be really useful to raise collisions and + // misses to users, somehow. if let (Some(dest), Some(target)) = (v4_dest, v4_target) { - out.insert(ResolvedVpcRoute { dest, target }); + out.insert(dest, target); } if let (Some(dest), Some(target)) = (v6_dest, v6_target) { - out.insert(ResolvedVpcRoute { dest, target }); + out.insert(dest, target); } } @@ -1775,6 +1780,7 @@ mod tests { use crate::db::datastore::test::sled_system_hardware_for_test; use crate::db::datastore::test_utils::datastore_test; use crate::db::datastore::test_utils::IneligibleSleds; + use crate::db::fixed_data::silo::DEFAULT_SILO; use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; use crate::db::model::Project; use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; @@ -2295,17 +2301,11 @@ mod tests { logctx.cleanup_successful(); } - // Test to verify that subnet CRUD operations are correctly - // reflected in the nexus-managed system router attached to a VPC. - #[tokio::test] - async fn test_vpc_system_router_sync_to_subnets() { - usdt::register_probes().unwrap(); - let logctx = - dev::test_setup_log("test_vpc_system_router_sync_to_subnets"); - let log = &logctx.log; - let mut db = test_setup_database(&logctx.log).await; - let (opctx, datastore) = datastore_test(&logctx, &db).await; - + async fn create_initial_vpc( + log: &slog::Logger, + opctx: &OpContext, + datastore: &DataStore, + ) -> (authz::Project, authz::Vpc, Vpc, authz::VpcRouter, VpcRouter) { // Create a project and VPC. let project_params = params::ProjectCreate { identity: IdentityMetadataCreateParams { @@ -2313,7 +2313,7 @@ mod tests { description: String::from("test project"), }, }; - let project = Project::new(Uuid::new_v4(), project_params); + let project = Project::new(DEFAULT_SILO.id(), project_params); let (authz_project, _) = datastore .project_create(&opctx, project) .await @@ -2368,28 +2368,30 @@ mod tests { }, ); - let (_, db_router) = datastore + let (authz_router, db_router) = datastore .vpc_create_router(&opctx, &authz_vpc, router) .await .unwrap(); - // InternetGateway route creation is handled by the saga proper, - // so we'll only have subnet routes here. Initially, we start with none: - verify_all_subnet_routes_in_router( - &opctx, - &datastore, - db_router.id(), - &[], - ) - .await; + (authz_project, authz_vpc, db_vpc, authz_router, db_router) + } - // Add a new subnet and we should get a new route. + async fn new_subnet_ez( + opctx: &OpContext, + datastore: &DataStore, + db_vpc: &Vpc, + authz_vpc: &authz::Vpc, + name: &str, + ip: [u8; 4], + prefix_len: u8, + ) -> (authz::VpcSubnet, VpcSubnet) { let ipv6_block = db_vpc .ipv6_prefix .random_subnet(external::Ipv6Net::VPC_SUBNET_IPV6_PREFIX_LENGTH) .map(|block| block.0) .unwrap(); - let (authz_sub0, sub0) = datastore + + datastore .vpc_create_subnet( &opctx, &authz_vpc, @@ -2397,13 +2399,13 @@ mod tests { Uuid::new_v4(), db_vpc.id(), IdentityMetadataCreateParams { - name: "s0".parse().unwrap(), - description: "The default subnet...".into(), + name: name.parse().unwrap(), + description: "A subnet...".into(), }, external::Ipv4Net( Ipv4Network::new( - core::net::Ipv4Addr::new(172, 30, 0, 0), - 22, + core::net::Ipv4Addr::from(ip), + prefix_len, ) .unwrap(), ), @@ -2411,7 +2413,45 @@ mod tests { ), ) .await - .unwrap(); + .unwrap() + } + + // Test to verify that subnet CRUD operations are correctly + // reflected in the nexus-managed system router attached to a VPC, + // and that these resolve to the v4/6 subnets of each. + #[tokio::test] + async fn test_vpc_system_router_sync_to_subnets() { + usdt::register_probes().unwrap(); + let logctx = + dev::test_setup_log("test_vpc_system_router_sync_to_subnets"); + let log = &logctx.log; + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + let (_, authz_vpc, db_vpc, _, db_router) = + create_initial_vpc(log, &opctx, &datastore).await; + + // InternetGateway route creation is handled by the saga proper, + // so we'll only have subnet routes here. Initially, we start with none: + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[], + ) + .await; + + // Add a new subnet and we should get a new route. + let (authz_sub0, sub0) = new_subnet_ez( + &opctx, + &datastore, + &db_vpc, + &authz_vpc, + "s0", + [172, 30, 0, 0], + 22, + ) + .await; verify_all_subnet_routes_in_router( &opctx, @@ -2422,34 +2462,16 @@ mod tests { .await; // Add another, and get another route. - let ipv6_block = db_vpc - .ipv6_prefix - .random_subnet(external::Ipv6Net::VPC_SUBNET_IPV6_PREFIX_LENGTH) - .map(|block| block.0) - .unwrap(); - let (_, sub1) = datastore - .vpc_create_subnet( - &opctx, - &authz_vpc, - db::model::VpcSubnet::new( - Uuid::new_v4(), - db_vpc.id(), - IdentityMetadataCreateParams { - name: "s1".parse().unwrap(), - description: "A second subnet...".into(), - }, - external::Ipv4Net( - Ipv4Network::new( - core::net::Ipv4Addr::new(172, 31, 0, 0), - 22, - ) - .unwrap(), - ), - ipv6_block, - ), - ) - .await - .unwrap(); + let (_, sub1) = new_subnet_ez( + &opctx, + &datastore, + &db_vpc, + &authz_vpc, + "s1", + [172, 31, 0, 0], + 22, + ) + .await; verify_all_subnet_routes_in_router( &opctx, @@ -2542,6 +2564,35 @@ mod tests { assert_eq!(count, 1, "subnet {name} should appear exactly once") } + // Resolve the routes: we should have two for each entry: + let resolved = datastore + .vpc_resolve_router_rules(&opctx, router_id) + .await + .unwrap(); + assert_eq!(resolved.len(), 2 * subnets.len()); + + // And each subnet generates a v4->v4 and v6->v6. + for subnet in subnets { + assert!(resolved.iter().any(|(k, v)| { + *k == subnet.ipv4_block.0.into() + && match v { + RouterTarget::VpcSubnet(ip) => { + *ip == subnet.ipv4_block.0.into() + } + _ => false, + } + })); + assert!(resolved.iter().any(|(k, v)| { + *k == subnet.ipv6_block.0.into() + && match v { + RouterTarget::VpcSubnet(ip) => { + *ip == subnet.ipv6_block.0.into() + } + _ => false, + } + })); + } + routes } @@ -2549,15 +2600,143 @@ mod tests { // of an instance NIC. #[tokio::test] async fn test_vpc_router_rule_instance_resolve() { - // use vpc_resolve_router_rules. - todo!() - } + usdt::register_probes().unwrap(); + let logctx = + dev::test_setup_log("test_vpc_router_rule_instance_resolve"); + let log = &logctx.log; + let db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; - // Test to verify that VPC routers resolve rules intelligently - // across dual IPv4 / IPv6 targets/destinations. - #[tokio::test] - async fn test_vpc_router_rule_v4_v6_resolve() { - // use vpc_resolve_router_rules. - todo!() + let (authz_project, authz_vpc, db_vpc, authz_router, _) = + create_initial_vpc(log, &opctx, &datastore).await; + + // Create a subnet for an instance to live in. + let (authz_sub0, sub0) = new_subnet_ez( + &opctx, + &datastore, + &db_vpc, + &authz_vpc, + "s0", + [172, 30, 0, 0], + 22, + ) + .await; + + // Add a rule pointing to the instance before it is created. + // We're commiting some minor data integrity sins by putting + // these into a system router, but that's irrelevant to resolution. + let inst_name = "insty".parse::().unwrap(); + let _ = datastore + .router_create_route( + &opctx, + &authz_router, + RouterRoute::new( + Uuid::new_v4(), + authz_router.id(), + external::RouterRouteKind::Custom, + params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "to-vpn".parse().unwrap(), + description: "A rule...".into(), + }, + target: external::RouteTarget::Instance( + inst_name.clone(), + ), + destination: external::RouteDestination::IpNet( + "192.168.0.0/16".parse().unwrap(), + ), + }, + ), + ) + .await + .unwrap(); + + // Resolve the rules: we will have two entries generated by the + // VPC subnet (v4, v6). + let routes = datastore + .vpc_resolve_router_rules(&opctx, authz_router.id()) + .await + .unwrap(); + + assert_eq!(routes.len(), 2); + + // Create an instance, this will have no effect for now as + // the instance lacks a NIC. + let db_inst = datastore + .project_create_instance( + &opctx, + &authz_project, + db::model::Instance::new( + Uuid::new_v4(), + authz_project.id(), + ¶ms::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: inst_name.clone(), + description: "An instance...".into(), + }, + ncpus: external::InstanceCpuCount(1), + memory: 10.into(), + hostname: "insty".parse().unwrap(), + user_data: vec![], + network_interfaces: + params::InstanceNetworkInterfaceAttachment::None, + external_ips: vec![], + disks: vec![], + ssh_public_keys: None, + start: false, + }, + ), + ) + .await + .unwrap(); + let (.., authz_instance) = + db::lookup::LookupPath::new(&opctx, &datastore) + .instance_id(db_inst.id()) + .lookup_for(authz::Action::CreateChild) + .await + .unwrap(); + + let routes = datastore + .vpc_resolve_router_rules(&opctx, authz_router.id()) + .await + .unwrap(); + + assert_eq!(routes.len(), 2); + + // Create a primary NIC on the instance; the route can now resolve + // to the instance's IP. + let nic = datastore + .instance_create_network_interface( + &opctx, + &authz_sub0, + &authz_instance, + IncompleteNetworkInterface::new_instance( + Uuid::new_v4(), + db_inst.id(), + sub0, + IdentityMetadataCreateParams { + name: "nic".parse().unwrap(), + description: "A NIC...".into(), + }, + None, + ) + .unwrap(), + ) + .await + .unwrap(); + + let routes = datastore + .vpc_resolve_router_rules(&opctx, authz_router.id()) + .await + .unwrap(); + + // Verify we now have a route pointing at this instance. + assert_eq!(routes.len(), 3); + assert!(routes.iter().any(|(k, v)| (*k + == "192.168.0.0/16".parse::().unwrap()) + && match v { + RouterTarget::Ip(ip) => *ip == nic.ip.ip(), + _ => false, + })); } } diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index f9b37d2175..f305990a22 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -238,8 +238,15 @@ impl BackgroundTask for VpcRouteManager { .await { Ok(rules) => { - set_rules(set.id, Some(version), rules.clone()); - known_rules.insert(router_id, rules); + let collapsed: HashSet<_> = rules + .into_iter() + .map(|(dest, target)| ResolvedVpcRoute { + dest, + target, + }) + .collect(); + set_rules(set.id, Some(version), collapsed.clone()); + known_rules.insert(router_id, collapsed); } Err(e) => { error!( From f02535e1e7fe82d6c93d19c64ead0c7e522d3bc8 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 24 May 2024 11:23:08 +0100 Subject: [PATCH 28/39] Accidentally ended up on the wrong maghemite. --- Cargo.lock | 8 ++++---- Cargo.toml | 4 ++-- workspace-hack/Cargo.toml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 728722907e..c9401f02cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" dependencies = [ "backtrace", ] @@ -1591,7 +1591,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=a557af774b2c3cfe1e9e4e27de9ad85de6a02a98#a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source = "git+https://github.com/oxidecomputer/maghemite?rev=c9824727eedc66d4920e42e7260df05050841ab8#c9824727eedc66d4920e42e7260df05050841ab8" dependencies = [ "percent-encoding", "progenitor", @@ -4295,7 +4295,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=a557af774b2c3cfe1e9e4e27de9ad85de6a02a98#a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source = "git+https://github.com/oxidecomputer/maghemite?rev=c9824727eedc66d4920e42e7260df05050841ab8#c9824727eedc66d4920e42e7260df05050841ab8" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index b70fbe25c9..ce52c84307 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -312,8 +312,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.12" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "c9824727eedc66d4920e42e7260df05050841ab8" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "c9824727eedc66d4920e42e7260df05050841ab8" } multimap = "0.10.0" nexus-client = { path = "clients/nexus-client" } nexus-config = { path = "nexus-config" } diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index d8c9e7c634..0ed8fbe17a 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -19,7 +19,7 @@ workspace = true [dependencies] ahash = { version = "0.8.11" } aho-corasick = { version = "1.1.3" } -anyhow = { version = "1.0.83", features = ["backtrace"] } +anyhow = { version = "1.0.86", features = ["backtrace"] } base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } @@ -124,7 +124,7 @@ zeroize = { version = "1.7.0", features = ["std", "zeroize_derive"] } [build-dependencies] ahash = { version = "0.8.11" } aho-corasick = { version = "1.1.3" } -anyhow = { version = "1.0.83", features = ["backtrace"] } +anyhow = { version = "1.0.86", features = ["backtrace"] } base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } From 880378ad372713ce780f6acd5945a6d58aae8d39 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 24 May 2024 13:46:14 +0100 Subject: [PATCH 29/39] Hook VPC checks into sim-sled-agent, instance networking tests. --- nexus/db-queries/src/db/datastore/silo.rs | 2 +- nexus/tests/integration_tests/instances.rs | 109 ++++++++++++++++++++- sled-agent/src/sim/http_entrypoints.rs | 30 +++++- sled-agent/src/sim/sled_agent.rs | 72 +++++++++++++- 4 files changed, 209 insertions(+), 4 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs index 76efcf99b0..0fd858b900 100644 --- a/nexus/db-queries/src/db/datastore/silo.rs +++ b/nexus/db-queries/src/db/datastore/silo.rs @@ -106,7 +106,7 @@ impl DataStore { Ok(()) } - pub(crate) async fn silo_create_query( + async fn silo_create_query( opctx: &OpContext, silo: Silo, ) -> Result, Error> { diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 51e2552e85..4b27c5038a 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -18,6 +18,7 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID; use nexus_db_queries::db::lookup::LookupPath; +use nexus_db_queries::db::DataStore; use nexus_test_interface::NexusServer; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; @@ -60,6 +61,8 @@ use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::Name; use omicron_common::api::external::NameOrId; use omicron_common::api::external::Vni; +use omicron_common::api::internal::shared::ResolvedVpcRoute; +use omicron_common::api::internal::shared::RouterId; use omicron_nexus::app::MAX_MEMORY_BYTES_PER_INSTANCE; use omicron_nexus::app::MAX_VCPU_PER_INSTANCE; use omicron_nexus::app::MIN_MEMORY_BYTES_PER_INSTANCE; @@ -68,6 +71,7 @@ use omicron_nexus::TestInterfaces as _; use omicron_sled_agent::sim::SledAgent; use omicron_test_utils::dev::poll::wait_for_condition; use sled_agent_client::TestInterfaces as _; +use std::collections::HashSet; use std::convert::TryFrom; use std::net::Ipv4Addr; use std::sync::Arc; @@ -670,6 +674,29 @@ async fn test_instance_start_creates_networking_state( for agent in &sled_agents { assert_sled_v2p_mappings(agent, &nics[0], guest_nics[0].vni).await; } + + // Ensure that the target sled agent for our instance has received + // up-to-date VPC routes. + let with_vmm = datastore + .instance_fetch_with_vmm(&opctx, &authz_instance) + .await + .unwrap(); + + let mut checked = false; + for agent in &sled_agents { + if Some(agent.id) == with_vmm.sled_id() { + assert_sled_vpc_routes( + agent, + &opctx, + datastore, + nics[0].subnet_id, + guest_nics[0].vni, + ) + .await; + checked = true; + } + } + assert!(checked); } #[nexus_test] @@ -769,7 +796,9 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { } #[nexus_test] -async fn test_instance_migrate_v2p(cptestctx: &ControlPlaneTestContext) { +async fn test_instance_migrate_v2p_and_routes( + cptestctx: &ControlPlaneTestContext, +) { let client = &cptestctx.external_client; let apictx = &cptestctx.server.server_context(); let nexus = &apictx.nexus; @@ -895,6 +924,15 @@ async fn test_instance_migrate_v2p(cptestctx: &ControlPlaneTestContext) { if sled_agent.id != dst_sled_id { assert_sled_v2p_mappings(sled_agent, &nics[0], guest_nics[0].vni) .await; + } else { + assert_sled_vpc_routes( + sled_agent, + &opctx, + datastore, + nics[0].subnet_id, + guest_nics[0].vni, + ) + .await; } } } @@ -4679,6 +4717,75 @@ async fn assert_sled_v2p_mappings( .expect("matching v2p mapping should be present"); } +/// Asserts that supplied sled agent's most recent VPC route sets +/// contain up-to-date routes for a known subnet. +pub async fn assert_sled_vpc_routes( + sled_agent: &Arc, + opctx: &OpContext, + datastore: &DataStore, + subnet_id: Uuid, + vni: Vni, +) { + let (.., authz_vpc, _, db_subnet) = LookupPath::new(opctx, datastore) + .vpc_subnet_id(subnet_id) + .fetch() + .await + .unwrap(); + + let custom_routes: HashSet<_> = + if let Some(router_id) = db_subnet.custom_router_id { + datastore + .vpc_resolve_router_rules(opctx, router_id) + .await + .unwrap() + .into_iter() + .map(|(dest, target)| ResolvedVpcRoute { dest, target }) + .collect() + } else { + Default::default() + }; + + let (.., vpc) = LookupPath::new(opctx, datastore) + .vpc_id(authz_vpc.id()) + .fetch() + .await + .unwrap(); + + let system_routes: HashSet<_> = datastore + .vpc_resolve_router_rules(opctx, vpc.system_router_id) + .await + .unwrap() + .into_iter() + .map(|(dest, target)| ResolvedVpcRoute { dest, target }) + .collect(); + + assert!(!system_routes.is_empty()); + + let condition = || async { + let vpc_routes = sled_agent.vpc_routes.lock().await; + let sys_routes_found = vpc_routes.iter().any(|(id, set)| { + *id == RouterId { vni, subnet: None } && set.routes == system_routes + }); + let custom_routes_found = vpc_routes.iter().any(|(id, set)| { + *id == RouterId { vni, subnet: Some(db_subnet.ipv4_block.0.into()) } + && set.routes == custom_routes + }); + + if sys_routes_found && custom_routes_found { + Ok(()) + } else { + Err(CondCheckError::NotYet::<()>) + } + }; + wait_for_condition( + condition, + &Duration::from_secs(1), + &Duration::from_secs(30), + ) + .await + .expect("matching vpc routes should be present"); +} + /// Simulate completion of an ongoing instance state transition. To do this, we /// have to look up the instance, then get the sled agent associated with that /// instance, and then tell it to finish simulating whatever async transition is diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index ae1318a8b1..977835e3ca 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -24,7 +24,9 @@ use illumos_utils::opte::params::VirtualNetworkInterfaceHost; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::UpdateArtifactId; -use omicron_common::api::internal::shared::SwitchPorts; +use omicron_common::api::internal::shared::{ + ResolvedVpcRouteSet, ResolvedVpcRouteState, SwitchPorts, +}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_storage::resources::DisksManagementResult; @@ -63,6 +65,8 @@ pub fn api() -> SledApiDescription { api.register(omicron_zones_get)?; api.register(omicron_zones_put)?; api.register(sled_add)?; + api.register(list_vpc_routes)?; + api.register(set_vpc_routes)?; Ok(()) } @@ -507,3 +511,27 @@ async fn sled_add( ) -> Result { Ok(HttpResponseUpdatedNoContent()) } + +#[endpoint { + method = GET, + path = "/vpc-routes", +}] +async fn list_vpc_routes( + rqctx: RequestContext>, +) -> Result>, HttpError> { + let sa = rqctx.context(); + Ok(HttpResponseOk(sa.list_vpc_routes().await)) +} + +#[endpoint { + method = PUT, + path = "/vpc-routes", +}] +async fn set_vpc_routes( + rqctx: RequestContext>, + body: TypedBody>, +) -> Result { + let sa = rqctx.context(); + sa.set_vpc_routes(body.into_inner()).await; + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index d9308bf769..9463e5468f 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -37,7 +37,10 @@ use omicron_common::api::internal::nexus::{ use omicron_common::api::internal::nexus::{ InstanceRuntimeState, VmmRuntimeState, }; -use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::{ + RackNetworkConfig, ResolvedVpcRoute, ResolvedVpcRouteSet, + ResolvedVpcRouteState, RouterId, RouterVersion, +}; use omicron_common::disk::DiskIdentity; use omicron_uuid_kinds::ZpoolUuid; use propolis_client::{ @@ -77,6 +80,7 @@ pub struct SledAgent { Mutex>, PropolisClient)>>, /// lists of external IPs assigned to instances pub external_ips: Mutex>>, + pub vpc_routes: Mutex>, config: Config, fake_zones: Mutex, instance_ensure_state_error: Mutex>, @@ -189,6 +193,7 @@ impl SledAgent { disk_id_to_region_ids: Mutex::new(HashMap::new()), v2p_mappings: Mutex::new(HashSet::new()), external_ips: Mutex::new(HashMap::new()), + vpc_routes: Mutex::new(HashMap::new()), mock_propolis: Mutex::new(None), config: config.clone(), fake_zones: Mutex::new(OmicronZonesConfig { @@ -358,6 +363,26 @@ impl SledAgent { self.map_disk_ids_to_region_ids(&vcr).await?; } + let mut routes = self.vpc_routes.lock().await; + for nic in &hardware.nics { + let my_routers = [ + RouterId { + // system + vni: nic.vni, + subnet: None, + }, + RouterId { + // custom + vni: nic.vni, + subnet: Some(nic.subnet), + }, + ]; + + for router in my_routers { + routes.entry(router).or_default(); + } + } + Ok(instance_run_time_state) } @@ -861,4 +886,49 @@ impl SledAgent { ) { *self.fake_zones.lock().await = requested_zones; } + + pub async fn list_vpc_routes(&self) -> Vec { + let routes = self.vpc_routes.lock().await; + routes + .iter() + .map(|(k, v)| ResolvedVpcRouteState { id: *k, version: v.version }) + .collect() + } + + pub async fn set_vpc_routes(&self, new_routes: Vec) { + let mut routes = self.vpc_routes.lock().await; + for new in new_routes { + // Disregard any route information for a subnet we don't have. + let Some(old) = routes.get(&new.id) else { + continue; + }; + + // We have to handle subnet router changes, as well as + // spurious updates from multiple Nexus instances. + // If there's a UUID match, only update if vers increased, + // otherwise take the update verbatim (including loss of version). + match (old.version, new.version) { + (Some(old_vers), Some(new_vers)) + if !old_vers.is_replaced_by(&new_vers) => + { + continue; + } + _ => {} + }; + + routes.insert( + new.id, + RouteSet { version: new.version, routes: new.routes }, + ); + } + } +} + +/// Stored routes (and usage count) for a given VPC/subnet. +// NB: We aren't doing post count tracking here to unsubscribe +// from (VNI, subnet) pairs. +#[derive(Debug, Clone, Default)] +pub struct RouteSet { + pub version: Option, + pub routes: HashSet, } From 08c982e38c9f22da4b2622e8a380244ce2ecf020 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 24 May 2024 14:40:52 +0100 Subject: [PATCH 30/39] Correctly cleanup after new tests... --- nexus/db-queries/src/db/datastore/vpc.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index a5ceb9a2c0..ae4db4813f 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -2604,7 +2604,7 @@ mod tests { let logctx = dev::test_setup_log("test_vpc_router_rule_instance_resolve"); let log = &logctx.log; - let db = test_setup_database(&logctx.log).await; + let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; let (authz_project, authz_vpc, db_vpc, authz_router, _) = @@ -2738,5 +2738,8 @@ mod tests { RouterTarget::Ip(ip) => *ip == nic.ip.ip(), _ => false, })); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); } } From e886d167b33a8991a2a17be43a4ed12fe586fd57 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 28 May 2024 17:43:45 +0100 Subject: [PATCH 31/39] Fix custom router listing. Very, very silly filter on the VPC ID in there... --- nexus/db-queries/src/db/datastore/vpc.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index ae4db4813f..ea3a7d5ca0 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1496,8 +1496,9 @@ impl DataStore { .eq(subnet_dsl::custom_router_id)), ) .filter(subnet_dsl::time_deleted.is_null()) - .filter(subnet_dsl::vpc_id.is_null()) + .filter(subnet_dsl::vpc_id.eq(vpc_id)) .filter(router_dsl::time_deleted.is_null()) + .filter(router_dsl::vpc_id.eq(vpc_id)) .select((VpcSubnet::as_select(), VpcRouter::as_select())) .load_async(&*self.pool_connection_authorized(opctx).await?) .await From cca764fea241ffe77f8c5aa6ba89874f6e17ad83 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 13 Jun 2024 14:02:36 +0100 Subject: [PATCH 32/39] Bump image. --- .github/buildomat/jobs/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 2dde4286dc..d15168d31f 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -2,7 +2,7 @@ #: #: name = "helios / deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.31" +#: target = "lab-2.0-opte-0.32" #: output_rules = [ #: "%/var/svc/log/oxide-sled-agent:default.log*", #: "%/zone/oxz_*/root/var/svc/log/oxide-*.log*", From 2425016f117891ae619619703bc6c3c94b63294c Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 13 Jun 2024 17:28:04 +0100 Subject: [PATCH 33/39] Forgot some maghemite SHAs... --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6c393a3c1..a8cbbc4307 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1591,7 +1591,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=c9824727eedc66d4920e42e7260df05050841ab8#c9824727eedc66d4920e42e7260df05050841ab8" +source = "git+https://github.com/oxidecomputer/maghemite?rev=e63f6d408908b3332d7cd89a4dd44a0f980d931d#e63f6d408908b3332d7cd89a4dd44a0f980d931d" dependencies = [ "percent-encoding", "progenitor", @@ -4275,7 +4275,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=c9824727eedc66d4920e42e7260df05050841ab8#c9824727eedc66d4920e42e7260df05050841ab8" +source = "git+https://github.com/oxidecomputer/maghemite?rev=e63f6d408908b3332d7cd89a4dd44a0f980d931d#e63f6d408908b3332d7cd89a4dd44a0f980d931d" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 1ba083ae47..0896ebc199 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -318,8 +318,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.12" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "c9824727eedc66d4920e42e7260df05050841ab8" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "c9824727eedc66d4920e42e7260df05050841ab8" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" } multimap = "0.10.0" nexus-auth = { path = "nexus/auth" } nexus-client = { path = "clients/nexus-client" } From 30e40437a0ad92133cadefe63cb6f7d0281d59a5 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 10:55:57 +0100 Subject: [PATCH 34/39] Minor fixes post-merge. --- nexus/db-queries/src/db/datastore/vpc.rs | 5 +++-- nexus/tests/integration_tests/instances.rs | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 6b33e2da8d..17e0c7ceb8 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1801,6 +1801,7 @@ mod tests { use omicron_common::api::external::Generation; use omicron_test_utils::dev; use omicron_uuid_kinds::GenericUuid; + use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::SledUuid; use oxnet::IpNet; use oxnet::Ipv4Net; @@ -2666,7 +2667,7 @@ mod tests { &opctx, &authz_project, db::model::Instance::new( - Uuid::new_v4(), + InstanceUuid::new_v4(), authz_project.id(), ¶ms::InstanceCreate { identity: IdentityMetadataCreateParams { @@ -2711,7 +2712,7 @@ mod tests { &authz_instance, IncompleteNetworkInterface::new_instance( Uuid::new_v4(), - db_inst.id(), + InstanceUuid::from_untyped_uuid(db_inst.id()), sub0, IdentityMetadataCreateParams { name: "nic".parse().unwrap(), diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index ed9b14a3e4..f7f970c625 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -686,7 +686,8 @@ async fn test_instance_start_creates_networking_state( let mut checked = false; for agent in &sled_agents { - if Some(agent.id) == with_vmm.sled_id() { + if Some(agent.id) == with_vmm.sled_id().map(SledUuid::into_untyped_uuid) + { assert_sled_vpc_routes( agent, &opctx, From 0057228378738ed5d8bab888d861fa6c1c6ad994 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 11:15:34 +0100 Subject: [PATCH 35/39] Review feedback. --- nexus/db-queries/src/db/datastore/vpc.rs | 82 +++++++++++++----------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 17e0c7ceb8..a265853cb5 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -46,6 +46,7 @@ use chrono::Utc; use diesel::prelude::*; use diesel::result::DatabaseErrorKind; use diesel::result::Error as DieselError; +use futures::stream::{self, StreamExt}; use ipnetwork::IpNetwork; use nexus_db_fixed_data::vpc::SERVICES_VPC_ID; use nexus_types::deployment::BlueprintZoneFilter; @@ -1582,51 +1583,58 @@ impl DataStore { } // TODO: This would be nice to solve in fewer queries. - let mut subnets = HashMap::new(); - for name in subnet_names.drain() { - if let Ok((.., subnet)) = db::lookup::LookupPath::new(opctx, self) - .vpc_id(authz_vpc.id()) - .vpc_subnet_name(Name::ref_cast(&name)) - .fetch() - .await - { - subnets.insert(name, subnet); - } - } - let mut vpcs = HashMap::new(); - for name in vpc_names.drain() { - if let Ok((.., vpc)) = db::lookup::LookupPath::new(opctx, self) - .project_id(authz_project.id()) - .vpc_name(Name::ref_cast(&name)) - .fetch() - .await - { - vpcs.insert(name, vpc); - } - } - let mut instances = HashMap::new(); - for name in instance_names.drain() { - if let Ok((.., authz_instance, instance)) = + let subnets = stream::iter(subnet_names) + .filter_map(|name| async { + db::lookup::LookupPath::new(opctx, self) + .vpc_id(authz_vpc.id()) + .vpc_subnet_name(Name::ref_cast(&name)) + .fetch() + .await + .ok() + .map(|(.., subnet)| (name, subnet)) + }) + .collect::>() + .await; + + // TODO: unused until VPC peering. + let _vpcs = stream::iter(vpc_names) + .filter_map(|name| async { + db::lookup::LookupPath::new(opctx, self) + .project_id(authz_project.id()) + .vpc_name(Name::ref_cast(&name)) + .fetch() + .await + .ok() + .map(|(.., vpc)| (name, vpc)) + }) + .collect::>() + .await; + + let instances = stream::iter(instance_names) + .filter_map(|name| async { db::lookup::LookupPath::new(opctx, self) .project_id(authz_project.id()) .instance_name(Name::ref_cast(&name)) .fetch() .await - { + .ok() + .map(|(.., auth, inst)| (name, auth, inst)) + }) + .filter_map(|(name, authz_instance, instance)| async move { // XXX: currently an instance can have one primary NIC, // and it is not dual-stack (v4 + v6). We need // to clarify what should be resolved in the v6 case. - if let Ok(primary_nic) = self - .instance_get_primary_network_interface( - opctx, - &authz_instance, - ) - .await - { - instances.insert(name, (instance, primary_nic)); - } - } - } + self.instance_get_primary_network_interface( + opctx, + &authz_instance, + ) + .await + .ok() + .map(|primary_nic| (name, (instance, primary_nic))) + }) + .collect::>() + .await; + // TODO: validate names of Internet Gateways. // See the discussion in `resolve_firewall_rules_for_sled_agent` on From 8162a2395ac7d897903bf9df88f842dd2c9948a7 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 26 Jun 2024 10:52:02 +0100 Subject: [PATCH 36/39] Review feedback: typed RouterKind instead of Option abuse --- common/src/api/internal/shared.rs | 12 +++++- illumos-utils/src/opte/port.rs | 8 +++- nexus/src/app/background/vpc_routes.rs | 12 ++++-- nexus/tests/integration_tests/instances.rs | 10 +++-- openapi/sled-agent.json | 47 ++++++++++++++++++---- sled-agent/src/sim/sled_agent.rs | 14 ++----- 6 files changed, 75 insertions(+), 28 deletions(-) diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 40cec46f8c..090b3c3058 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -672,7 +672,17 @@ impl RouterVersion { )] pub struct RouterId { pub vni: Vni, - pub subnet: Option, + pub kind: RouterKind, +} + +/// The scope of a set of VPC router rules. +#[derive( + Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +#[serde(tag = "type", rename_all = "snake_case", content = "subnet")] +pub enum RouterKind { + System, + Custom(IpNet), } /// Version information for routes on a given VPC subnet. diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 352411c49c..a692a02304 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -9,6 +9,7 @@ use crate::opte::Vni; use macaddr::MacAddr6; use omicron_common::api::external; use omicron_common::api::internal::shared::RouterId; +use omicron_common::api::internal::shared::RouterKind; use oxnet::IpNet; use std::net::IpAddr; use std::sync::Arc; @@ -140,10 +141,13 @@ impl Port { pub fn system_router_key(&self) -> RouterId { // Unwrap safety: both of these VNI types represent validated u24s. let vni = external::Vni::try_from(self.vni().as_u32()).unwrap(); - RouterId { vni, subnet: None } + RouterId { vni, kind: RouterKind::System } } pub fn custom_router_key(&self) -> RouterId { - RouterId { subnet: Some(*self.subnet()), ..self.system_router_key() } + RouterId { + kind: RouterKind::Custom(*self.subnet()), + ..self.system_router_key() + } } } diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index f305990a22..6500b13b5b 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -15,7 +15,7 @@ use nexus_types::{ identity::Resource, }; use omicron_common::api::internal::shared::{ - ResolvedVpcRoute, ResolvedVpcRouteSet, RouterId, RouterVersion, + ResolvedVpcRoute, ResolvedVpcRouteSet, RouterId, RouterKind, RouterVersion, }; use serde_json::json; use std::collections::hash_map::Entry; @@ -164,7 +164,7 @@ impl BackgroundTask for VpcRouteManager { }; db_routers.insert( - RouterId { vni: set.id.vni, subnet: None }, + RouterId { vni: set.id.vni, kind: RouterKind::System }, system_router, ); db_routers.extend(custom_routers.iter().map( @@ -172,7 +172,9 @@ impl BackgroundTask for VpcRouteManager { ( RouterId { vni: set.id.vni, - subnet: Some(subnet.ipv4_block.0.into()), + kind: RouterKind::Custom( + subnet.ipv4_block.0.into(), + ), }, router.clone(), ) @@ -183,7 +185,9 @@ impl BackgroundTask for VpcRouteManager { ( RouterId { vni: set.id.vni, - subnet: Some(subnet.ipv6_block.0.into()), + kind: RouterKind::Custom( + subnet.ipv6_block.0.into(), + ), }, router, ) diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index f7f970c625..75ddf847bf 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -62,6 +62,7 @@ use omicron_common::api::external::NameOrId; use omicron_common::api::external::Vni; use omicron_common::api::internal::shared::ResolvedVpcRoute; use omicron_common::api::internal::shared::RouterId; +use omicron_common::api::internal::shared::RouterKind; use omicron_nexus::app::MAX_MEMORY_BYTES_PER_INSTANCE; use omicron_nexus::app::MAX_VCPU_PER_INSTANCE; use omicron_nexus::app::MIN_MEMORY_BYTES_PER_INSTANCE; @@ -4850,11 +4851,14 @@ pub async fn assert_sled_vpc_routes( let condition = || async { let vpc_routes = sled_agent.vpc_routes.lock().await; let sys_routes_found = vpc_routes.iter().any(|(id, set)| { - *id == RouterId { vni, subnet: None } && set.routes == system_routes + *id == RouterId { vni, kind: RouterKind::System } + && set.routes == system_routes }); let custom_routes_found = vpc_routes.iter().any(|(id, set)| { - *id == RouterId { vni, subnet: Some(db_subnet.ipv4_block.0.into()) } - && set.routes == custom_routes + *id == RouterId { + vni, + kind: RouterKind::Custom(db_subnet.ipv4_block.0.into()), + } && set.routes == custom_routes }); if sys_routes_found && custom_routes_found { diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 823425afd3..4f2bcd6e98 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -4382,22 +4382,55 @@ "description": "Identifier for a VPC and/or subnet.", "type": "object", "properties": { - "subnet": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] + "kind": { + "$ref": "#/components/schemas/RouterKind" }, "vni": { "$ref": "#/components/schemas/Vni" } }, "required": [ + "kind", "vni" ] }, + "RouterKind": { + "description": "The scope of a set of VPC router rules.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "system" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "subnet": { + "$ref": "#/components/schemas/IpNet" + }, + "type": { + "type": "string", + "enum": [ + "custom" + ] + } + }, + "required": [ + "subnet", + "type" + ] + } + ] + }, "RouterTarget": { "description": "The target for a given router entry.", "oneOf": [ diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 1b7211e7d8..9cb146531b 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -38,7 +38,7 @@ use omicron_common::api::internal::nexus::{ }; use omicron_common::api::internal::shared::{ RackNetworkConfig, ResolvedVpcRoute, ResolvedVpcRouteSet, - ResolvedVpcRouteState, RouterId, RouterVersion, + ResolvedVpcRouteState, RouterId, RouterKind, RouterVersion, }; use omicron_common::disk::DiskIdentity; use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid, ZpoolUuid}; @@ -368,16 +368,8 @@ impl SledAgent { let mut routes = self.vpc_routes.lock().await; for nic in &hardware.nics { let my_routers = [ - RouterId { - // system - vni: nic.vni, - subnet: None, - }, - RouterId { - // custom - vni: nic.vni, - subnet: Some(nic.subnet), - }, + RouterId { vni: nic.vni, kind: RouterKind::System }, + RouterId { vni: nic.vni, kind: RouterKind::Custom(nic.subnet) }, ]; for router in my_routers { From e1971bc9d6a74a290d63203d22a3f6cc74a44de2 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 26 Jun 2024 11:30:06 +0100 Subject: [PATCH 37/39] Bump Maghemite. --- Cargo.lock | 21 +++++++++++---------- Cargo.toml | 6 +++--- clients/ddm-admin-client/src/lib.rs | 9 ++++----- openapi/bootstrap-agent.json | 10 ++++++++++ openapi/nexus-internal.json | 10 ++++++++++ openapi/nexus.json | 1 + openapi/sled-agent.json | 10 ++++++++++ openapi/wicketd.json | 12 ++++++++++++ package-manifest.toml | 12 ++++++------ tools/maghemite_ddm_openapi_version | 4 ++-- tools/maghemite_mg_openapi_version | 4 ++-- tools/maghemite_mgd_checksums | 4 ++-- workspace-hack/Cargo.toml | 8 ++++---- 13 files changed, 77 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ab08c0e3c..8ee031fe24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1605,8 +1605,9 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=e63f6d408908b3332d7cd89a4dd44a0f980d931d#e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source = "git+https://github.com/oxidecomputer/maghemite?rev=3c3fa8482fe09a01da62fbd35efe124ea9cac9e7#3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" dependencies = [ + "oxnet", "percent-encoding", "progenitor", "reqwest", @@ -4007,7 +4008,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.52.5", + "windows-targets 0.48.5", ] [[package]] @@ -4290,7 +4291,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=e63f6d408908b3332d7cd89a4dd44a0f980d931d#e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source = "git+https://github.com/oxidecomputer/maghemite?rev=3c3fa8482fe09a01da62fbd35efe124ea9cac9e7#3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" dependencies = [ "anyhow", "chrono", @@ -6411,7 +6412,7 @@ dependencies = [ [[package]] name = "oxnet" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/oxnet?branch=main#42b4d3c77c7f5f2636cd6c4bbf37ac3eada047e0" +source = "git+https://github.com/oxidecomputer/oxnet#2612d2203effcfdcbf83778a77f1bfd03fe6ed24" dependencies = [ "ipnetwork", "schemars", @@ -8287,9 +8288,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0218ceea14babe24a4a5836f86ade86c1effbc198164e619194cb5069187e29" +checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" dependencies = [ "bytes", "chrono", @@ -8302,9 +8303,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed5a1ccce8ff962e31a165d41f6e2a2dd1245099dc4d594f5574a86cd90f4d3" +checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" dependencies = [ "proc-macro2", "quote", @@ -8489,9 +8490,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" dependencies = [ "itoa", "ryu", diff --git a/Cargo.toml b/Cargo.toml index 4c83e9229c..bf3a8c1feb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -328,8 +328,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.12" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" } multimap = "0.10.0" nexus-auth = { path = "nexus/auth" } nexus-client = { path = "clients/nexus-client" } @@ -349,7 +349,7 @@ omicron-certificates = { path = "certificates" } omicron-passwords = { path = "passwords" } omicron-workspace-hack = "0.1.0" oxlog = { path = "dev-tools/oxlog" } -oxnet = { git = "https://github.com/oxidecomputer/oxnet", branch = "main" } +oxnet = { git = "https://github.com/oxidecomputer/oxnet" } nexus-test-interface = { path = "nexus/test-interface" } nexus-test-utils-macros = { path = "nexus/test-utils-macros" } nexus-test-utils = { path = "nexus/test-utils" } diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index b926ee2971..8cd9781e1d 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -12,7 +12,7 @@ pub use ddm_admin_client::types; pub use ddm_admin_client::Error; -use ddm_admin_client::types::{Ipv6Prefix, TunnelOrigin}; +use ddm_admin_client::types::TunnelOrigin; use ddm_admin_client::Client as InnerClient; use either::Either; use omicron_common::address::Ipv6Subnet; @@ -81,8 +81,7 @@ impl Client { pub fn advertise_prefix(&self, address: Ipv6Subnet) { let me = self.clone(); tokio::spawn(async move { - let prefix = - Ipv6Prefix { addr: address.net().prefix(), len: SLED_PREFIX }; + let prefix = address.net(); retry_notify(retry_policy_internal_service_aggressive(), || async { info!( me.log, "Sending prefix to ddmd for advertisement"; @@ -130,8 +129,8 @@ impl Client { let prefixes = self.inner.get_prefixes().await?.into_inner(); Ok(prefixes.into_iter().flat_map(|(_, prefixes)| { prefixes.into_iter().flat_map(|prefix| { - let mut segments = prefix.destination.addr.segments(); - if prefix.destination.len == BOOTSTRAP_MASK + let mut segments = prefix.destination.addr().segments(); + if prefix.destination.width() == BOOTSTRAP_MASK && segments[0] == BOOTSTRAP_PREFIX { Either::Left(interfaces.iter().map(move |interface| { diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 5d175e7b09..6050939b94 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -328,6 +328,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -340,6 +341,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -437,6 +439,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -444,11 +447,13 @@ "md5_auth_key": { "nullable": true, "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": "string" }, "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -456,6 +461,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -467,6 +473,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -474,6 +481,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -1192,6 +1200,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -1234,6 +1243,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index c3cc3c059d..72731e83e8 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -1567,6 +1567,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -1579,6 +1580,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -1676,6 +1678,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1683,11 +1686,13 @@ "md5_auth_key": { "nullable": true, "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": "string" }, "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -1695,6 +1700,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1706,6 +1712,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1713,6 +1720,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -4345,6 +4353,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -5003,6 +5012,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/openapi/nexus.json b/openapi/nexus.json index 3e0fe9d75c..a17ccf1b84 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -16002,6 +16002,7 @@ "signing_keypair": { "nullable": true, "description": "request signing key pair", + "default": null, "allOf": [ { "$ref": "#/components/schemas/DerEncodedKeyPair" diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 4f2bcd6e98..3ac130c565 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -1519,6 +1519,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -1531,6 +1532,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -1628,6 +1630,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1635,11 +1638,13 @@ "md5_auth_key": { "nullable": true, "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": "string" }, "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -1647,6 +1652,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1658,6 +1664,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1665,6 +1672,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -4368,6 +4376,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -4817,6 +4826,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 21e8ebeedd..555b8cf44c 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -1049,6 +1049,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -1061,6 +1062,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -2854,6 +2856,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -5050,6 +5053,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -5070,6 +5074,7 @@ }, "allowed_export": { "description": "Apply export policy to this peer with an allow list.", + "default": null, "allOf": [ { "$ref": "#/components/schemas/UserSpecifiedImportExportPolicy" @@ -5078,6 +5083,7 @@ }, "allowed_import": { "description": "Apply import policy to this peer with an allow list.", + "default": null, "allOf": [ { "$ref": "#/components/schemas/UserSpecifiedImportExportPolicy" @@ -5093,6 +5099,7 @@ "auth_key_id": { "nullable": true, "description": "The key identifier for authentication to use with the peer.", + "default": null, "allOf": [ { "$ref": "#/components/schemas/BgpAuthKeyId" @@ -5152,6 +5159,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -5159,6 +5167,7 @@ "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -5166,6 +5175,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -5177,6 +5187,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -5184,6 +5195,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/package-manifest.toml b/package-manifest.toml index 30fc288766..797be3e4b3 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -548,10 +548,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source.commit = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "fb1a9c24f160afb3f87b57544869f2b54876b2b4bb2f6922411e2eb04ecd4a61" +source.sha256 = "63b6c74584e32f52893730e3a567da29c7f93934c38882614aad59034bdd980d" output.type = "tarball" [package.mg-ddm] @@ -564,10 +564,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source.commit = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "07a534a6dd6975cd13c629414382f6fdb8d0a11406dd626ec93c6d50d0e7e041" +source.sha256 = "b9908b81fee00d71b750f5b9a0f866c807adb0f924ab635295d28753538836f5" output.type = "zone" output.intermediate_only = true @@ -579,10 +579,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source.commit = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "cf7b30a650a6501030743485f29bd556e6a379333bac2b89e746e2d389423ded" +source.sha256 = "51f446933f0d8c426b15ea0845b66664da9b9a129893d12b25d7912b52f07362" output.type = "zone" output.intermediate_only = true diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index ed24b94811..569d3d7813 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="e63f6d408908b3332d7cd89a4dd44a0f980d931d" -SHA2="004e873e4120aa26460271368485266b75b7f964e5ed4dbee8fb5db4519470d7" +COMMIT="3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" +SHA2="007bfb717ccbc077c0250dee3121aeb0c5bb0d1c16795429a514fa4f8635a5ef" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index e334871b3e..de64133971 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="e63f6d408908b3332d7cd89a4dd44a0f980d931d" -SHA2="fdb33ee7425923560534672264008ef8948d227afce948ab704de092ad72157c" +COMMIT="3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" +SHA2="e4b42ab9daad90f0c561a830b62a9d17e294b4d0da0a6d44b4030929b0c37b7e" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 2577af45d3..f9d4fd4491 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="cf7b30a650a6501030743485f29bd556e6a379333bac2b89e746e2d389423ded" -MGD_LINUX_SHA256="1d5421c3229d8a4a512ba1ca35620bb9f235c7ae80518a78589214bb16ed5148" +CIDL_SHA256="51f446933f0d8c426b15ea0845b66664da9b9a129893d12b25d7912b52f07362" +MGD_LINUX_SHA256="736067394778cc4c38fecb1ca8647db3ca7ab1b5c4446f3ce2b5350379ba95b7" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 1b21b72495..70730e8a76 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -89,11 +89,11 @@ regex-automata = { version = "0.4.6", default-features = false, features = ["dfa regex-syntax = { version = "0.8.3" } reqwest = { version = "0.11.27", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] } ring = { version = "0.17.8", features = ["std"] } -schemars = { version = "0.8.20", features = ["bytes", "chrono", "uuid1"] } +schemars = { version = "0.8.21", features = ["bytes", "chrono", "uuid1"] } scopeguard = { version = "1.2.0" } semver = { version = "1.0.23", features = ["serde"] } serde = { version = "1.0.203", features = ["alloc", "derive", "rc"] } -serde_json = { version = "1.0.117", features = ["raw_value", "unbounded_depth"] } +serde_json = { version = "1.0.118", features = ["raw_value", "unbounded_depth"] } sha2 = { version = "0.10.8", features = ["oid"] } similar = { version = "2.5.0", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } @@ -193,11 +193,11 @@ regex-automata = { version = "0.4.6", default-features = false, features = ["dfa regex-syntax = { version = "0.8.3" } reqwest = { version = "0.11.27", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] } ring = { version = "0.17.8", features = ["std"] } -schemars = { version = "0.8.20", features = ["bytes", "chrono", "uuid1"] } +schemars = { version = "0.8.21", features = ["bytes", "chrono", "uuid1"] } scopeguard = { version = "1.2.0" } semver = { version = "1.0.23", features = ["serde"] } serde = { version = "1.0.203", features = ["alloc", "derive", "rc"] } -serde_json = { version = "1.0.117", features = ["raw_value", "unbounded_depth"] } +serde_json = { version = "1.0.118", features = ["raw_value", "unbounded_depth"] } sha2 = { version = "0.10.8", features = ["oid"] } similar = { version = "2.5.0", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } From 70263a2f3c662f5c699f40e473148f0102feecd2 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 26 Jun 2024 12:40:06 +0100 Subject: [PATCH 38/39] Better conflict resolution on Nexus-managed subnet route names --- nexus/db-model/src/vpc_route.rs | 27 ++++++-- nexus/db-queries/src/db/datastore/vpc.rs | 80 ++++++++++++++---------- schema/crdb/vpc-subnet-routing/up03.sql | 8 +-- 3 files changed, 73 insertions(+), 42 deletions(-) diff --git a/nexus/db-model/src/vpc_route.rs b/nexus/db-model/src/vpc_route.rs index dda7f0b785..3015df691f 100644 --- a/nexus/db-model/src/vpc_route.rs +++ b/nexus/db-model/src/vpc_route.rs @@ -128,13 +128,32 @@ impl RouterRoute { } } + /// Create a subnet routing rule for a VPC's system router. + /// + /// This defaults to use the same name as the subnet. If this would conflict + /// with the internet gateway rules, then the UUID is used instead (alongside + /// notice that a name conflict has occurred). pub fn for_subnet( route_id: Uuid, system_router_id: Uuid, subnet: Name, - ) -> Result { - let name = format!("sn-{}", subnet).parse().map_err(|_| ())?; - Ok(Self::new( + ) -> Self { + let forbidden_names = ["default-v4", "default-v6"]; + + let name = if forbidden_names.contains(&subnet.as_str()) { + // unwrap safety: a uuid is not by itself a valid name + // so prepend it with another string. + // - length constraint is <63 chars, + // - a UUID is 36 chars including hyphens, + // - "{subnet}-" is 11 chars + // - "conflict-" is 9 chars + // = 56 chars + format!("conflict-{subnet}-{route_id}").parse().unwrap() + } else { + subnet.0.clone() + }; + + Self::new( route_id, system_router_id, external::RouterRouteKind::VpcSubnet, @@ -146,7 +165,7 @@ impl RouterRoute { target: external::RouteTarget::Subnet(subnet.0.clone()), destination: external::RouteDestination::Subnet(subnet.0), }, - )) + ) } } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index a265853cb5..988f40e770 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -296,8 +296,8 @@ impl DataStore { route_id, SERVICES_VPC.system_router_id, vpc_subnet.name().clone().into(), - ) - .expect("builtin service names are short enough for route naming"); + ); + self.router_create_route(opctx, &authz_router, route) .await .map(|_| ()) @@ -1326,15 +1326,11 @@ impl DataStore { // modify other system routes like internet gateways (which are // `RouteKind::Default`). let conn = self.pool_connection_authorized(opctx).await?; - let log = opctx.log.clone(); self.transaction_retry_wrapper("vpc_subnet_route_reconcile") - .transaction(&conn, |conn| { - let log = log.clone(); - async move { - + .transaction(&conn, |conn| async move { use db::schema::router_route::dsl; - use db::schema::vpc_subnet::dsl as subnet; use db::schema::vpc::dsl as vpc; + use db::schema::vpc_subnet::dsl as subnet; let system_router_id = vpc::vpc .filter(vpc::id.eq(vpc_id)) @@ -1352,7 +1348,10 @@ impl DataStore { .await?; let current_rules: Vec = dsl::router_route - .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .filter( + dsl::kind + .eq(RouterRouteKind(ExternalRouteKind::VpcSubnet)), + ) .filter(dsl::time_deleted.is_null()) .filter(dsl::vpc_router_id.eq(system_router_id)) .select(RouterRoute::as_select()) @@ -1360,18 +1359,28 @@ impl DataStore { .await?; // Build the add/delete sets. - let expected_names: HashSet = valid_subnets.iter() + let expected_names: HashSet = valid_subnets + .iter() .map(|v| v.identity.name.clone()) .collect(); + // This checks that we have rules which *point to* the named + // subnets, rather than working with rule names (even if these + // are set to match the subnet where possible). + // Rule names are effectively randomised when someone, e.g., + // names a subnet "default-v4"/"-v6", and this prevents us + // from repeatedly adding/deleting that route. let mut found_names = HashSet::new(); let mut invalid = Vec::new(); for rule in current_rules { let id = rule.id(); match (rule.kind.0, rule.target.0) { - (ExternalRouteKind::VpcSubnet, RouteTarget::Subnet(n)) - if expected_names.contains(Name::ref_cast(&n)) => - {let _ = found_names.insert(n.into());}, + ( + ExternalRouteKind::VpcSubnet, + RouteTarget::Subnet(n), + ) if expected_names.contains(Name::ref_cast(&n)) => { + let _ = found_names.insert(n.into()); + } _ => invalid.push(id), } } @@ -1394,34 +1403,34 @@ impl DataStore { // Duplicate rules are caught here using the UNIQUE constraint // on names in a router. Only nexus can alter the system router, // so there is no risk of collision with user-specified names. + // + // Subnets named "default-v4" or "default-v6" have their rules renamed + // to include the rule UUID. for subnet in expected_names.difference(&found_names) { let route_id = Uuid::new_v4(); - // XXX this is fallible as it is based on subnet name. - // need to control this somewhere sane. - let Ok(route) = db::model::RouterRoute::for_subnet( + let route = db::model::RouterRoute::for_subnet( route_id, system_router_id, subnet.clone(), - ) else { - error!( - log, - "Reconciling VPC routes: name {} in vpc {} is too long", - subnet, - vpc_id, - ); - continue; - }; - - match Self::router_create_route_on_connection(route, &conn).await { - Err(Error::Conflict { .. }) => return Err(DieselError::RollbackTransaction), + ); + + match Self::router_create_route_on_connection(route, &conn) + .await + { + Err(Error::Conflict { .. }) => { + return Err(DieselError::RollbackTransaction) + } Err(_) => return Err(DieselError::NotFound), - _ => {}, + _ => {} } } // Verify that route set is exactly as intended, and rollback otherwise. let current_rules: Vec = dsl::router_route - .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .filter( + dsl::kind + .eq(RouterRouteKind(ExternalRouteKind::VpcSubnet)), + ) .filter(dsl::time_deleted.is_null()) .filter(dsl::vpc_router_id.eq(system_router_id)) .select(RouterRoute::as_select()) @@ -1429,19 +1438,22 @@ impl DataStore { .await?; if current_rules.len() != expected_names.len() { - return Err(DieselError::RollbackTransaction) + return Err(DieselError::RollbackTransaction); } for rule in current_rules { match (rule.kind.0, rule.target.0) { - (ExternalRouteKind::VpcSubnet, RouteTarget::Subnet(n)) - if expected_names.contains(Name::ref_cast(&n)) => {}, + ( + ExternalRouteKind::VpcSubnet, + RouteTarget::Subnet(n), + ) if expected_names.contains(Name::ref_cast(&n)) => {} _ => return Err(DieselError::RollbackTransaction), } } Ok(()) - }}).await + }) + .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; self.vpc_increment_rpw_version(opctx, vpc_id).await diff --git a/schema/crdb/vpc-subnet-routing/up03.sql b/schema/crdb/vpc-subnet-routing/up03.sql index d256921d34..7c4cc97a80 100644 --- a/schema/crdb/vpc-subnet-routing/up03.sql +++ b/schema/crdb/vpc-subnet-routing/up03.sql @@ -51,7 +51,7 @@ INSERT INTO omicron.public.router_route target, destination ) SELECT - gen_random_uuid(), 'sn-' || vpc_subnet.name, + gen_random_uuid(), vpc_subnet.name, 'VPC Subnet route for ''' || vpc_subnet.name || '''', now(), now(), omicron.public.vpc_router.id, 'default', @@ -76,15 +76,15 @@ WITH known_ids (new_id, new_name, new_description) AS ( 'Default internet gateway route for Oxide Services' ), ( - '001de000-c470-4000-8000-000000000004', 'sn-external-dns', + '001de000-c470-4000-8000-000000000004', 'external-dns', 'Built-in VPC Subnet for Oxide service (external-dns)' ), ( - '001de000-c470-4000-8000-000000000005', 'sn-nexus', + '001de000-c470-4000-8000-000000000005', 'nexus', 'Built-in VPC Subnet for Oxide service (nexus)' ), ( - '001de000-c470-4000-8000-000000000006', 'sn-boundary-ntp', + '001de000-c470-4000-8000-000000000006', 'boundary-ntp', 'Built-in VPC Subnet for Oxide service (boundary-ntp)' ) ) From 62de75eb7cbb19d088593a6245a70c236cf02ddf Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 26 Jun 2024 20:09:01 +0100 Subject: [PATCH 39/39] Unearthed a nice li'l bug during migration on london --- nexus/db-queries/src/db/datastore/vpc.rs | 46 +++++++++++++++++++++++- schema/crdb/vpc-subnet-routing/up03.sql | 2 +- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 988f40e770..89ee1c468e 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -2482,7 +2482,7 @@ mod tests { .await; // Add another, and get another route. - let (_, sub1) = new_subnet_ez( + let (authz_sub1, sub1) = new_subnet_ez( &opctx, &datastore, &db_vpc, @@ -2536,6 +2536,50 @@ mod tests { ) .await; + // If we use a reserved name, we should be able to update the table. + let sub1 = datastore + .vpc_update_subnet( + &opctx, + &authz_sub1, + VpcSubnetUpdate { + name: Some( + "default-v4".parse::().unwrap().into(), + ), + description: None, + time_modified: Utc::now(), + }, + ) + .await + .unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub1], + ) + .await; + + // Ditto for adding such a route. + let (_, sub0) = new_subnet_ez( + &opctx, + &datastore, + &db_vpc, + &authz_vpc, + "default-v6", + [172, 30, 0, 0], + 22, + ) + .await; + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub0, &sub1], + ) + .await; + db.cleanup().await.unwrap(); logctx.cleanup_successful(); } diff --git a/schema/crdb/vpc-subnet-routing/up03.sql b/schema/crdb/vpc-subnet-routing/up03.sql index 7c4cc97a80..fb4fd2324a 100644 --- a/schema/crdb/vpc-subnet-routing/up03.sql +++ b/schema/crdb/vpc-subnet-routing/up03.sql @@ -54,7 +54,7 @@ SELECT gen_random_uuid(), vpc_subnet.name, 'VPC Subnet route for ''' || vpc_subnet.name || '''', now(), now(), - omicron.public.vpc_router.id, 'default', + omicron.public.vpc_router.id, 'vpc_subnet', 'subnet:' || vpc_subnet.name, 'subnet:' || vpc_subnet.name FROM (omicron.public.vpc_subnet JOIN omicron.public.vpc