From 76106b11c08949563507cc0a86bd6bde520457e2 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 23 Apr 2024 12:17:18 +0100 Subject: [PATCH 01/59] Clear XDE underlay when destroying virtual hardware OPTE now prevents itself from being unloaded if its underlay state is set. Currently, underlay setup is performed only once, and it seems to be the case that XDE can be unloaded in some scenarios (e.g., `a4x2` setup). However, a consequence is that removing the driver requires an extra operation to explicitly clear the underlay state. This PR adds this operation to the `cargo xtask virtual-hardware destroy` command. This is currently blocked on opte#485 being approved/merged. Closes #5314. --- Cargo.lock | 14 +++++++------- Cargo.toml | 4 ++-- dev-tools/xtask/src/virtual_hardware.rs | 12 +++++++++++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f5477f077..46c1f4de60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1750,7 +1750,7 @@ dependencies = [ [[package]] name = "derror-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "darling", "proc-macro2", @@ -3467,7 +3467,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" [[package]] name = "illumos-utils" @@ -3874,7 +3874,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "quote", "syn 2.0.59", @@ -5953,7 +5953,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "cfg-if", "derror-macro", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -5983,7 +5983,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -6057,7 +6057,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=7ee353a470ea59529ee1b34729681da887aa88ce#7ee353a470ea59529ee1b34729681da887aa88ce" +source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index b6b937614c..df14c579d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -305,14 +305,14 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb", features = [ "api", "std" ] } once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "7ee353a470ea59529ee1b34729681da887aa88ce" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb" } oso = "0.27" owo-colors = "4.0.0" oximeter = { path = "oximeter/oximeter" } diff --git a/dev-tools/xtask/src/virtual_hardware.rs b/dev-tools/xtask/src/virtual_hardware.rs index 95190ebfde..c480dc355c 100644 --- a/dev-tools/xtask/src/virtual_hardware.rs +++ b/dev-tools/xtask/src/virtual_hardware.rs @@ -100,6 +100,7 @@ const IPADM: &'static str = "/usr/sbin/ipadm"; const MODINFO: &'static str = "/usr/sbin/modinfo"; const MODUNLOAD: &'static str = "/usr/sbin/modunload"; const NETSTAT: &'static str = "/usr/bin/netstat"; +const OPTEADM: &'static str = "/opt/oxide/opte/bin/opteadm"; const PFEXEC: &'static str = "/usr/bin/pfexec"; const PING: &'static str = "/usr/sbin/ping"; const SWAP: &'static str = "/usr/sbin/swap"; @@ -240,8 +241,17 @@ fn unload_xde_driver() -> Result<()> { println!("xde driver already unloaded"); return Ok(()); }; - println!("unloading xde driver"); + println!("unloading xde driver:\na) clearing underlay..."); + let mut cmd = Command::new(PFEXEC); + cmd.args([OPTEADM, "clear-xde-underlay"]); + if let Err(e) = execute(cmd) { + // This is explicitly non-fatal: the underlay is only set when + // sled-agent is running. We still need to be able to tear + // down the driver if we immediately call create->destroy. + println!("\tFailed or already unset: {e}"); + } + println!("b) unloading module..."); let mut cmd = Command::new(PFEXEC); cmd.arg(MODUNLOAD); cmd.arg("-i"); From 94e180808689303d5a9896a347ea858766701e46 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 21 May 2024 12:27:27 +0100 Subject: [PATCH 02/59] Pull in latest OPTE after merge. --- .github/buildomat/jobs/a4x2-deploy.sh | 2 +- .github/buildomat/jobs/deploy.sh | 2 +- Cargo.lock | 28 ++++++++------------------- Cargo.toml | 8 ++++---- package-manifest.toml | 16 +++++++-------- tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 4 ++-- tools/maghemite_mgd_checksums | 4 ++-- tools/opte_version | 2 +- 9 files changed, 28 insertions(+), 40 deletions(-) diff --git a/.github/buildomat/jobs/a4x2-deploy.sh b/.github/buildomat/jobs/a4x2-deploy.sh index 323b3e2e28..53153beafb 100755 --- a/.github/buildomat/jobs/a4x2-deploy.sh +++ b/.github/buildomat/jobs/a4x2-deploy.sh @@ -2,7 +2,7 @@ #: #: name = "a4x2-deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.27" +#: target = "lab-2.0-opte-0.31" #: output_rules = [ #: "/out/falcon/*.log", #: "/out/falcon/*.err", diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index c947a05e10..bee2700d89 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -2,7 +2,7 @@ #: #: name = "helios / deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.28" +#: target = "lab-2.0-opte-0.31" #: output_rules = [ #: "%/var/svc/log/oxide-sled-agent:default.log*", #: "%/zone/oxz_*/root/var/svc/log/oxide-*.log*", diff --git a/Cargo.lock b/Cargo.lock index 2ec770ff83..ad3ac965d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1591,7 +1591,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=025389ff39d594bf2b815377e2c1dc4dd23b1f96#025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source = "git+https://github.com/oxidecomputer/maghemite?rev=a557af774b2c3cfe1e9e4e27de9ad85de6a02a98#a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" dependencies = [ "percent-encoding", "progenitor", @@ -1729,17 +1729,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derror-macro" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn 2.0.64", -] - [[package]] name = "dhcproto" version = "0.11.0" @@ -3481,7 +3470,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" [[package]] name = "illumos-utils" @@ -3894,7 +3883,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "quote", "syn 2.0.64", @@ -4306,7 +4295,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=025389ff39d594bf2b815377e2c1dc4dd23b1f96#025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source = "git+https://github.com/oxidecomputer/maghemite?rev=a557af774b2c3cfe1e9e4e27de9ad85de6a02a98#a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" dependencies = [ "anyhow", "chrono", @@ -6019,10 +6008,9 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "cfg-if", - "derror-macro", "dyn-clone", "illumos-sys-hdrs", "kstat-macro", @@ -6037,7 +6025,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -6049,7 +6037,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -6123,7 +6111,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=bca9fd01b0116a558c429d6929542b910b0cd5bb#bca9fd01b0116a558c429d6929542b910b0cd5bb" +source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 8d3c80f6c5..8d298d1feb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -312,8 +312,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.12" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" } multimap = "0.10.0" nexus-client = { path = "clients/nexus-client" } nexus-config = { path = "nexus-config" } @@ -347,14 +347,14 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504", features = [ "api", "std" ] } once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "bca9fd01b0116a558c429d6929542b910b0cd5bb" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504" } oso = "0.27" owo-colors = "4.0.0" oximeter = { path = "oximeter/oximeter" } diff --git a/package-manifest.toml b/package-manifest.toml index 2bfc51d533..096cd6f011 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -533,10 +533,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" # The SHA256 digest is automatically posted to: -# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt -source.sha256 = "f2ee54b6a654daa1c1f817440317e9b11c5ddc71249df261bb5cfa0e6057dc24" +# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt +source.sha256 = "b7a88a985bb29c105fffa184c9826e3243e87d3063987b90b39e51abcd8b8526" output.type = "tarball" [package.mg-ddm] @@ -549,10 +549,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "bb98815f759f38abee9f5aea0978cd33e66e75079cc8c171036be21bf9049c96" +source.sha256 = "8d0556bcf83360df653f72e885cf28e628601f17137d9475a234b1020c6366ba" output.type = "zone" output.intermediate_only = true @@ -564,10 +564,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" +source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" # The SHA256 digest is automatically posted to: -# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "e0907de39ca9f8ab45d40d361a1dbeed4bd8e9b157f8d3d8fe0a4bc259d933bd" +# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt +source.sha256 = "91fb4c893240e6f781ab6c3d6f2af8c0e9378fd646d1e45233502e3a4d23cb1a" output.type = "zone" output.intermediate_only = true diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index c39c9690bb..6cbb69ad79 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="025389ff39d594bf2b815377e2c1dc4dd23b1f96" +COMMIT="a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" SHA2="004e873e4120aa26460271368485266b75b7f964e5ed4dbee8fb5db4519470d7" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 966e4de7fe..fe46171c0d 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="025389ff39d594bf2b815377e2c1dc4dd23b1f96" -SHA2="a5d2f275c99152711dec1df58fd49d459d3fcb8fbfc7a7f48f432be248d74639" +COMMIT="a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +SHA2="fdb33ee7425923560534672264008ef8948d227afce948ab704de092ad72157c" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index eeb873a424..70471ccf6d 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="e0907de39ca9f8ab45d40d361a1dbeed4bd8e9b157f8d3d8fe0a4bc259d933bd" -MGD_LINUX_SHA256="903413ddaab89594ed7518cb8f2f27793e96cd17ed2d6b3fe11657ec4375cb19" +CIDL_SHA256="91fb4c893240e6f781ab6c3d6f2af8c0e9378fd646d1e45233502e3a4d23cb1a" +MGD_LINUX_SHA256="111c111691bb42e5fde4d5a3d2a613ffc7e7e57c536866f4f00fb5afb60c6da4" diff --git a/tools/opte_version b/tools/opte_version index e1b3e11499..fc3e603c41 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.28.233 +0.31.258 From b67f416773f69306920539ac6104c2ace0c6ee82 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 21 May 2024 12:55:18 +0100 Subject: [PATCH 03/59] Bump OPTE and related. --- Cargo.lock | 12 ++++++------ Cargo.toml | 4 ++-- package-manifest.toml | 12 ++++++------ tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 2 +- tools/maghemite_mgd_checksums | 4 ++-- tools/opte_version | 2 +- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad3ac965d0..728722907e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3470,7 +3470,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" [[package]] name = "illumos-utils" @@ -3883,7 +3883,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "quote", "syn 2.0.64", @@ -6008,7 +6008,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "cfg-if", "dyn-clone", @@ -6025,7 +6025,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -6037,7 +6037,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -6111,7 +6111,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=a0294dbf886bcaf76d33b5c8f0ca6616a9b90781#a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 8d298d1feb..b70fbe25c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -347,14 +347,14 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "a0294dbf886bcaf76d33b5c8f0ca6616a9b90781", features = [ "api", "std" ] } once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" } oso = "0.27" owo-colors = "4.0.0" oximeter = { path = "oximeter/oximeter" } diff --git a/package-manifest.toml b/package-manifest.toml index 096cd6f011..e86587245d 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -533,10 +533,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source.commit = "c9824727eedc66d4920e42e7260df05050841ab8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "b7a88a985bb29c105fffa184c9826e3243e87d3063987b90b39e51abcd8b8526" +source.sha256 = "4ba6bb87fe9bd2dcb69b27a218e0f9f10767fa8eb3c1439fbf0fa5f5a1921dd9" output.type = "tarball" [package.mg-ddm] @@ -549,10 +549,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source.commit = "c9824727eedc66d4920e42e7260df05050841ab8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "8d0556bcf83360df653f72e885cf28e628601f17137d9475a234b1020c6366ba" +source.sha256 = "45918b2ef4fe2be048cc8934155335a1b20e91fea6d818c385fe3964a3e8fdec" output.type = "zone" output.intermediate_only = true @@ -564,10 +564,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source.commit = "c9824727eedc66d4920e42e7260df05050841ab8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "91fb4c893240e6f781ab6c3d6f2af8c0e9378fd646d1e45233502e3a4d23cb1a" +source.sha256 = "6ae4bc3b332e91706c1c6633a7fc218aac65b7feff5643ee2dbbe79b841e0df3" output.type = "zone" output.intermediate_only = true diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 6cbb69ad79..edd1b0d670 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +COMMIT="c9824727eedc66d4920e42e7260df05050841ab8" SHA2="004e873e4120aa26460271368485266b75b7f964e5ed4dbee8fb5db4519470d7" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index fe46171c0d..efe080571c 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +COMMIT="c9824727eedc66d4920e42e7260df05050841ab8" SHA2="fdb33ee7425923560534672264008ef8948d227afce948ab704de092ad72157c" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 70471ccf6d..d2ad05383d 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="91fb4c893240e6f781ab6c3d6f2af8c0e9378fd646d1e45233502e3a4d23cb1a" -MGD_LINUX_SHA256="111c111691bb42e5fde4d5a3d2a613ffc7e7e57c536866f4f00fb5afb60c6da4" +CIDL_SHA256="6ae4bc3b332e91706c1c6633a7fc218aac65b7feff5643ee2dbbe79b841e0df3" +MGD_LINUX_SHA256="7930008cf8ce535a8b31043fc3edde0e825bd54d75f73234929bd0037ecc3a41" diff --git a/tools/opte_version b/tools/opte_version index fc3e603c41..a8f4dc593f 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.31.258 +0.32.262 From 5f7cfa8a49205ffb4fa86ffd7288054787f6ffe1 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 21 May 2024 13:02:41 +0100 Subject: [PATCH 04/59] Compat with newer OPTE, more believable router rules --- common/src/api/external/mod.rs | 3 + illumos-utils/src/opte/port_manager.rs | 9 ++ nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/schema_versions.rs | 3 +- nexus/db-model/src/vpc_subnet.rs | 3 + nexus/db-queries/src/db/datastore/vpc.rs | 98 +++++++++---- nexus/db-queries/src/db/fixed_data/vpc.rs | 18 ++- .../src/db/fixed_data/vpc_subnet.rs | 18 +++ nexus/src/app/sagas/vpc_create.rs | 133 ++++++++++++++++-- nexus/src/external_api/http_entrypoints.rs | 12 +- nexus/types/src/external_api/views.rs | 5 +- schema/crdb/dbinit.sql | 7 +- schema/crdb/vpc-subnet-routing/up01.sql | 3 + 13 files changed, 251 insertions(+), 62 deletions(-) create mode 100644 schema/crdb/vpc-subnet-routing/up01.sql diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 1c01782cc6..f385c0b4fa 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1680,6 +1680,9 @@ pub enum RouteTarget { #[display("inetgw:{0}")] /// Forward traffic to an internet gateway InternetGateway(Name), + #[display("drop")] + /// Drop matching traffic + Drop, } /// A `RouteDestination` is used to match traffic with a routing rule, on the diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 03c51c321d..d6b4963607 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -25,6 +25,7 @@ use oxide_vpc::api::IpCidr; use oxide_vpc::api::Ipv4Cfg; use oxide_vpc::api::Ipv6Cfg; use oxide_vpc::api::MacAddr; +use oxide_vpc::api::RouterClass; use oxide_vpc::api::RouterTarget; use oxide_vpc::api::SNat4Cfg; use oxide_vpc::api::SNat6Cfg; @@ -339,9 +340,16 @@ impl PortManager { (port, ticket) }; + // TODO: These should not be filled in like this, and should be informed + // by either our existing knowledge of current knowledge of system + custom + // routers OR we just await the router RPW filling this in for us. + // In future, ∃ VPCs *without* an Internet Gateway so we can't just + // plumb that in as well... + // Add a router entry for this interface's subnet, directing traffic to the // VPC subnet. let route = AddRouterEntryReq { + class: RouterClass::System, port_name: port_name.clone(), dest: vpc_subnet, target: RouterTarget::VpcSubnet(vpc_subnet), @@ -378,6 +386,7 @@ impl PortManager { .parse() .unwrap(); let route = AddRouterEntryReq { + class: RouterClass::System, port_name: port_name.clone(), dest, target: RouterTarget::InternetGateway, diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 224c461da0..95d372167e 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1099,6 +1099,7 @@ table! { rcgen -> Int8, ipv4_block -> Inet, ipv6_block -> Inet, + custom_router_id -> Nullable, } } diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index cb229274fe..47a5689d07 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(63, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(64, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(64, "vpc-subnet-routing"), KnownVersion::new(63, "remove-producer-base-route-column"), KnownVersion::new(62, "allocate-subnet-decommissioned-sleds"), KnownVersion::new(61, "blueprint-add-sled-state"), diff --git a/nexus/db-model/src/vpc_subnet.rs b/nexus/db-model/src/vpc_subnet.rs index 407c933ef2..5c85c8a6dc 100644 --- a/nexus/db-model/src/vpc_subnet.rs +++ b/nexus/db-model/src/vpc_subnet.rs @@ -39,6 +39,7 @@ pub struct VpcSubnet { pub rcgen: Generation, pub ipv4_block: Ipv4Net, pub ipv6_block: Ipv6Net, + pub custom_router_id: Option, } impl VpcSubnet { @@ -60,6 +61,7 @@ impl VpcSubnet { rcgen: Generation::new(), ipv4_block: Ipv4Net(ipv4_block), ipv6_block: Ipv6Net(ipv6_block), + custom_router_id: None, } } @@ -102,6 +104,7 @@ impl From for views::VpcSubnet { vpc_id: subnet.vpc_id, ipv4_block: subnet.ipv4_block.0, ipv6_block: subnet.ipv6_block.0, + custom_router_id: subnet.custom_router_id, } } } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 91843abf2e..d0c4e381ad 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -74,7 +74,8 @@ impl DataStore { ) -> Result<(), Error> { use crate::db::fixed_data::project::SERVICES_PROJECT_ID; use crate::db::fixed_data::vpc::SERVICES_VPC; - use crate::db::fixed_data::vpc::SERVICES_VPC_DEFAULT_ROUTE_ID; + use crate::db::fixed_data::vpc::SERVICES_VPC_DEFAULT_V4_ROUTE_ID; + use crate::db::fixed_data::vpc::SERVICES_VPC_DEFAULT_V6_ROUTE_ID; opctx.authorize(authz::Action::Modify, &authz::DATABASE).await?; @@ -135,35 +136,41 @@ impl DataStore { .map(|(authz_router, _)| authz_router)? }; - let route = RouterRoute::new( - *SERVICES_VPC_DEFAULT_ROUTE_ID, - SERVICES_VPC.system_router_id, - RouterRouteKind::Default, - nexus_types::external_api::params::RouterRouteCreate { - identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), - description: - "Default internet gateway route for Oxide Services" - .to_string(), + // Unwrap safety: these are known valid CIDR blocks. + let default_ips = [ + ("0.0.0.0/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V4_ROUTE_ID), + ("::/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V6_ROUTE_ID), + ]; + + for (default, uuid) in default_ips { + let route = RouterRoute::new( + uuid, + SERVICES_VPC.system_router_id, + RouterRouteKind::Default, + nexus_types::external_api::params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: + "Default internet gateway route for Oxide Services" + .to_string(), + }, + target: RouteTarget::InternetGateway( + "outbound".parse().unwrap(), + ), + destination: RouteDestination::IpNet(default), }, - target: RouteTarget::InternetGateway( - "outbound".parse().unwrap(), - ), - destination: RouteDestination::Vpc( - SERVICES_VPC.identity.name.clone().into(), - ), - }, - ); - self.router_create_route(opctx, &authz_router, route) - .await - .map(|_| ()) - .or_else(|e| match e { - Error::ObjectAlreadyExists { .. } => Ok(()), - _ => Err(e), - })?; + ); + self.router_create_route(opctx, &authz_router, route) + .await + .map(|_| ()) + .or_else(|e| match e { + Error::ObjectAlreadyExists { .. } => Ok(()), + _ => Err(e), + })?; + } self.load_builtin_vpc_fw_rules(opctx).await?; - self.load_builtin_vpc_subnets(opctx).await?; + self.load_builtin_vpc_subnets(opctx, &authz_router).await?; info!(opctx.log, "created built-in services vpc"); @@ -228,10 +235,14 @@ impl DataStore { async fn load_builtin_vpc_subnets( &self, opctx: &OpContext, + authz_router: &authz::VpcRouter, ) -> Result<(), Error> { use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET; + use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET_ROUTE_ID; use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; + use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET_ROUTE_ID; use crate::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET; + use crate::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET_ROUTE_ID; debug!(opctx.log, "attempting to create built-in VPC Subnets"); @@ -242,9 +253,11 @@ impl DataStore { .lookup_for(authz::Action::CreateChild) .await .internal_context("lookup built-in services vpc")?; - for vpc_subnet in - [&*DNS_VPC_SUBNET, &*NEXUS_VPC_SUBNET, &*NTP_VPC_SUBNET] - { + for (vpc_subnet, route_id) in [ + (&*DNS_VPC_SUBNET, *DNS_VPC_SUBNET_ROUTE_ID), + (&*NEXUS_VPC_SUBNET, *NEXUS_VPC_SUBNET_ROUTE_ID), + (&*NTP_VPC_SUBNET, *NTP_VPC_SUBNET_ROUTE_ID), + ] { if let Ok(_) = db::lookup::LookupPath::new(opctx, self) .vpc_subnet_id(vpc_subnet.id()) .fetch() @@ -260,6 +273,31 @@ impl DataStore { Error::ObjectAlreadyExists { .. } => Ok(()), _ => Err(e), })?; + + let route = RouterRoute::new( + route_id, + *SERVICES_VPC_ID, + RouterRouteKind::Default, + nexus_types::external_api::params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: + "Default internet gateway route for Oxide Services" + .to_string(), + }, + target: RouteTarget::Subnet(vpc_subnet.name().clone()), + destination: RouteDestination::Subnet( + vpc_subnet.name().clone(), + ), + }, + ); + self.router_create_route(opctx, &authz_router, route) + .await + .map(|_| ()) + .or_else(|e| match e { + Error::ObjectAlreadyExists { .. } => Ok(()), + _ => Err(e), + })?; } info!(opctx.log, "created built-in services vpc subnets"); diff --git a/nexus/db-queries/src/db/fixed_data/vpc.rs b/nexus/db-queries/src/db/fixed_data/vpc.rs index c71b655ddc..6dffc11426 100644 --- a/nexus/db-queries/src/db/fixed_data/vpc.rs +++ b/nexus/db-queries/src/db/fixed_data/vpc.rs @@ -24,11 +24,19 @@ pub static SERVICES_VPC_ROUTER_ID: Lazy = Lazy::new(|| { }); /// UUID of default route for built-in Services VPC. -pub static SERVICES_VPC_DEFAULT_ROUTE_ID: Lazy = Lazy::new(|| { - "001de000-074c-4000-8000-000000000002" - .parse() - .expect("invalid uuid for builtin services vpc default route id") -}); +pub static SERVICES_VPC_DEFAULT_V4_ROUTE_ID: Lazy = + Lazy::new(|| { + "001de000-074c-4000-8000-000000000002" + .parse() + .expect("invalid uuid for builtin services vpc default route id") + }); + +pub static SERVICES_VPC_DEFAULT_V6_ROUTE_ID: Lazy = + Lazy::new(|| { + "001de000-074c-4000-8000-000000000003" + .parse() + .expect("invalid uuid for builtin services vpc default route id") + }); /// Built-in VPC for internal services on the rack. pub static SERVICES_VPC: Lazy = Lazy::new(|| { diff --git a/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs b/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs index c42d4121c9..45db9b7e0b 100644 --- a/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs +++ b/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs @@ -31,6 +31,24 @@ pub static NTP_VPC_SUBNET_ID: Lazy = Lazy::new(|| { .expect("invalid uuid for builtin boundary ntp vpc subnet id") }); +pub static DNS_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { + "001de000-c470-4000-8000-000000000004" + .parse() + .expect("invalid uuid for builtin services vpc default route id") +}); + +pub static NEXUS_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { + "001de000-c470-4000-8000-000000000005" + .parse() + .expect("invalid uuid for builtin services vpc default route id") +}); + +pub static NTP_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { + "001de000-c470-4000-8000-000000000006" + .parse() + .expect("invalid uuid for builtin services vpc default route id") +}); + /// Built-in VPC Subnet for External DNS. pub static DNS_VPC_SUBNET: Lazy = Lazy::new(|| { VpcSubnet::new( diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index fdd117b850..647ee05da5 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -13,6 +13,7 @@ use nexus_db_queries::{authn, authz, db}; use nexus_defaults as defaults; use omicron_common::api::external; use omicron_common::api::external::IdentityMetadataCreateParams; +use omicron_common::api::external::IpNet; use omicron_common::api::external::LookupType; use omicron_common::api::external::RouteDestination; use omicron_common::api::external::RouteTarget; @@ -44,14 +45,22 @@ declare_saga_actions! { + svc_create_router - svc_create_router_undo } - VPC_CREATE_ROUTE -> "route" { - + svc_create_route - - svc_create_route_undo + VPC_CREATE_V4_ROUTE -> "route4" { + + svc_create_v4_route + - svc_create_v4_route_undo + } + VPC_CREATE_V6_ROUTE -> "route6" { + + svc_create_v6_route + - svc_create_v6_route_undo } VPC_CREATE_SUBNET -> "subnet" { + svc_create_subnet - svc_create_subnet_undo } + VPC_CREATE_SUBNET_ROUTE -> "route" { + + svc_create_subnet_route + - svc_create_subnet_route_undo + } VPC_UPDATE_FIREWALL -> "firewall" { + svc_update_firewall - svc_update_firewall_undo @@ -79,8 +88,18 @@ pub fn create_dag( ACTION_GENERATE_ID.as_ref(), )); builder.append(Node::action( - "default_route_id", - "GenerateDefaultRouteId", + "default_v4_route_id", + "GenerateDefaultV4RouteId", + ACTION_GENERATE_ID.as_ref(), + )); + builder.append(Node::action( + "default_v6_route_id", + "GenerateDefaultV6RouteId", + ACTION_GENERATE_ID.as_ref(), + )); + builder.append(Node::action( + "default_subnet_route_id", + "GenerateDefaultV6RouteId", ACTION_GENERATE_ID.as_ref(), )); builder.append(Node::action( @@ -90,8 +109,10 @@ pub fn create_dag( )); builder.append(vpc_create_vpc_action()); builder.append(vpc_create_router_action()); - builder.append(vpc_create_route_action()); + builder.append(vpc_create_v4_route_action()); + builder.append(vpc_create_v6_route_action()); builder.append(vpc_create_subnet_action()); + builder.append(vpc_create_subnet_route_action()); builder.append(vpc_update_firewall_action()); builder.append(vpc_notify_sleds_action()); @@ -217,8 +238,44 @@ async fn svc_create_router_undo( Ok(()) } +// XX: possibly do these as a subsaga? + +async fn svc_create_v4_route( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let default_route_id = sagactx.lookup::("default_v4_route_id")?; + let default_route = + "0.0.0.0/0".parse().expect("known-valid specifier for a default route"); + svc_create_route(sagactx, default_route_id, default_route).await +} + +async fn svc_create_v4_route_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let route_id = sagactx.lookup::("default_v4_route_id")?; + svc_create_route_undo(sagactx, route_id).await +} + +async fn svc_create_v6_route( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let default_route_id = sagactx.lookup::("default_v6_route_id")?; + let default_route = + "::/0".parse().expect("known-valid specifier for a default route"); + svc_create_route(sagactx, default_route_id, default_route).await +} + +async fn svc_create_v6_route_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let route_id = sagactx.lookup::("default_v6_route_id")?; + svc_create_route_undo(sagactx, route_id).await +} + async fn svc_create_route( sagactx: NexusActionContext, + route_id: Uuid, + default_net: IpNet, ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; @@ -226,12 +283,11 @@ async fn svc_create_route( &sagactx, ¶ms.serialized_authn, ); - let default_route_id = sagactx.lookup::("default_route_id")?; let system_router_id = sagactx.lookup::("system_router_id")?; let authz_router = sagactx.lookup::("router")?; let route = db::model::RouterRoute::new( - default_route_id, + route_id, system_router_id, RouterRouteKind::Default, params::RouterRouteCreate { @@ -240,9 +296,7 @@ async fn svc_create_route( description: "The default route of a vpc".to_string(), }, target: RouteTarget::InternetGateway("outbound".parse().unwrap()), - destination: RouteDestination::Vpc( - params.vpc_create.identity.name.clone(), - ), + destination: RouteDestination::IpNet(default_net), }, ); @@ -256,6 +310,7 @@ async fn svc_create_route( async fn svc_create_route_undo( sagactx: NexusActionContext, + route_id: Uuid, ) -> Result<(), anyhow::Error> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; @@ -264,7 +319,6 @@ async fn svc_create_route_undo( ¶ms.serialized_authn, ); let authz_router = sagactx.lookup::("router")?; - let route_id = sagactx.lookup::("default_route_id")?; let authz_route = authz::RouterRoute::new( authz_router, route_id, @@ -370,6 +424,61 @@ async fn svc_create_subnet_undo( Ok(()) } +async fn svc_create_subnet_route( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let system_router_id = sagactx.lookup::("system_router_id")?; + let authz_router = sagactx.lookup::("router")?; + let route_id = sagactx.lookup::("default_subnet_route_id")?; + + let route = db::model::RouterRoute::new( + route_id, + system_router_id, + RouterRouteKind::Default, + params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: "The default route of a vpc".to_string(), + }, + target: RouteTarget::Subnet("default".parse().unwrap()), + destination: RouteDestination::Subnet("default".parse().unwrap()), + }, + ); + + osagactx + .datastore() + .router_create_route(&opctx, &authz_router, route) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + +async fn svc_create_subnet_route_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let authz_router = sagactx.lookup::("router")?; + let route_id = sagactx.lookup::("default_subnet_route_id")?; + let authz_route = authz::RouterRoute::new( + authz_router, + route_id, + LookupType::ById(route_id), + ); + osagactx.datastore().router_delete_route(&opctx, &authz_route).await?; + Ok(()) +} + async fn svc_update_firewall( sagactx: NexusActionContext, ) -> Result, ActionError> { diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 350836441e..2678768b48 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -5446,7 +5446,6 @@ async fn vpc_firewall_rules_update( method = GET, path = "/v1/vpc-routers", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_list( rqctx: RequestContext, @@ -5486,7 +5485,6 @@ async fn vpc_router_list( method = GET, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_view( rqctx: RequestContext, @@ -5520,7 +5518,6 @@ async fn vpc_router_view( method = POST, path = "/v1/vpc-routers", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_create( rqctx: RequestContext, @@ -5556,7 +5553,6 @@ async fn vpc_router_create( method = DELETE, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_delete( rqctx: RequestContext, @@ -5590,7 +5586,6 @@ async fn vpc_router_delete( method = PUT, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_update( rqctx: RequestContext, @@ -5630,7 +5625,6 @@ async fn vpc_router_update( method = GET, path = "/v1/vpc-router-routes", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_list( rqctx: RequestContext, @@ -5672,7 +5666,6 @@ async fn vpc_router_route_list( method = GET, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_view( rqctx: RequestContext, @@ -5704,12 +5697,11 @@ async fn vpc_router_route_view( .await } -/// Create router +/// Create route #[endpoint { method = POST, path = "/v1/vpc-router-routes", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_create( rqctx: RequestContext, @@ -5745,7 +5737,6 @@ async fn vpc_router_route_create( method = DELETE, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_delete( rqctx: RequestContext, @@ -5781,7 +5772,6 @@ async fn vpc_router_route_delete( method = PUT, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_update( rqctx: RequestContext, diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 1e90d04b55..cb05f85930 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -258,7 +258,7 @@ pub struct Vpc { } /// A VPC subnet represents a logical grouping for instances that allows network traffic between -/// them, within a IPv4 subnetwork or optionall an IPv6 subnetwork. +/// them, within a IPv4 subnetwork or optionally an IPv6 subnetwork. #[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct VpcSubnet { /// common identifying metadata @@ -273,6 +273,9 @@ pub struct VpcSubnet { /// The IPv6 subnet CIDR block. pub ipv6_block: Ipv6Net, + + /// ID for an attached custom router. + pub custom_router_id: Option, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index cc298e4565..486ae68240 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -1367,7 +1367,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.vpc_subnet ( /* Child resource creation generation number */ rcgen INT8 NOT NULL, ipv4_block INET NOT NULL, - ipv6_block INET NOT NULL + ipv6_block INET NOT NULL, + /* nullable FK to the `vpc_router` table. */ + custom_router_id UUID ); /* Subnet and network interface names are unique per VPC, not project */ @@ -1623,6 +1625,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.router_route ( /* Indicates that the object has been deleted */ time_deleted TIMESTAMPTZ, + /* FK to the `vpc_router` table. */ vpc_router_id UUID NOT NULL, kind omicron.public.router_route_kind NOT NULL, target STRING(128) NOT NULL, @@ -3859,7 +3862,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '63.0.0', NULL) + (TRUE, NOW(), NOW(), '64.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/vpc-subnet-routing/up01.sql b/schema/crdb/vpc-subnet-routing/up01.sql new file mode 100644 index 0000000000..d1869dd010 --- /dev/null +++ b/schema/crdb/vpc-subnet-routing/up01.sql @@ -0,0 +1,3 @@ +-- Each subnet may have a custom router attached. +ALTER TABLE omicron.public.vpc_subnet +ADD COLUMN IF NOT EXISTS custom_router_id UUID; From 16e0107f71774d3ef429f76f2174d99aeb77c9c5 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 15 May 2024 14:39:43 +0100 Subject: [PATCH 05/59] VPC Subnet route reconcile. --- nexus/db-model/src/vpc_route.rs | 23 +++- nexus/db-queries/src/db/datastore/vpc.rs | 167 ++++++++++++++++++++--- nexus/src/app/sagas/vpc_create.rs | 26 ++-- 3 files changed, 179 insertions(+), 37 deletions(-) diff --git a/nexus/db-model/src/vpc_route.rs b/nexus/db-model/src/vpc_route.rs index 168ed41cef..2c561325a5 100644 --- a/nexus/db-model/src/vpc_route.rs +++ b/nexus/db-model/src/vpc_route.rs @@ -18,7 +18,7 @@ use std::io::Write; use uuid::Uuid; impl_enum_wrapper!( - #[derive(SqlType, Debug)] + #[derive(SqlType, Debug, QueryId)] #[diesel(postgres_type(name = "router_route_kind", schema = "public"))] pub struct RouterRouteKindEnum; @@ -127,6 +127,27 @@ impl RouterRoute { destination: RouteDestination::new(params.destination), } } + + pub fn for_subnet( + route_id: Uuid, + system_router_id: Uuid, + subnet: Name, + ) -> Result { + let name = format!("subnet_{}", subnet).parse().map_err(|_| ())?; + Ok(Self::new( + route_id, + system_router_id, + external::RouterRouteKind::VpcSubnet, + params::RouterRouteCreate { + identity: external::IdentityMetadataCreateParams { + name, + description: format!("VPC Subnet route for '{subnet}'"), + }, + target: external::RouteTarget::Subnet(subnet.0.clone()), + destination: external::RouteDestination::Subnet(subnet.0), + }, + )) + } } impl Into for RouterRoute { diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index d0c4e381ad..029f46e3c1 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -21,6 +21,7 @@ use crate::db::model::InstanceNetworkInterface; use crate::db::model::Name; use crate::db::model::Project; use crate::db::model::RouterRoute; +use crate::db::model::RouterRouteKind; use crate::db::model::RouterRouteUpdate; use crate::db::model::Sled; use crate::db::model::Vni; @@ -59,11 +60,12 @@ use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_common::api::external::RouteDestination; use omicron_common::api::external::RouteTarget; -use omicron_common::api::external::RouterRouteKind; +use omicron_common::api::external::RouterRouteKind as ExternalRouteKind; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::Vni as ExternalVni; use ref_cast::RefCast; use std::collections::BTreeMap; +use std::collections::HashSet; use uuid::Uuid; impl DataStore { @@ -146,7 +148,7 @@ impl DataStore { let route = RouterRoute::new( uuid, SERVICES_VPC.system_router_id, - RouterRouteKind::Default, + ExternalRouteKind::Default, nexus_types::external_api::params::RouterRouteCreate { identity: IdentityMetadataCreateParams { name: "default".parse().unwrap(), @@ -274,23 +276,12 @@ impl DataStore { _ => Err(e), })?; - let route = RouterRoute::new( + let route = RouterRoute::for_subnet( route_id, *SERVICES_VPC_ID, - RouterRouteKind::Default, - nexus_types::external_api::params::RouterRouteCreate { - identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), - description: - "Default internet gateway route for Oxide Services" - .to_string(), - }, - target: RouteTarget::Subnet(vpc_subnet.name().clone()), - destination: RouteDestination::Subnet( - vpc_subnet.name().clone(), - ), - }, - ); + vpc_subnet.name().clone().into(), + ) + .expect("builtin service names are short enough for route naming"); self.router_create_route(opctx, &authz_router, route) .await .map(|_| ()) @@ -808,6 +799,9 @@ impl DataStore { assert_eq!(authz_vpc.id(), subnet.vpc_id); let db_subnet = self.vpc_create_subnet_raw(subnet).await?; + self.vpc_system_router_ensure_subnet_routes(opctx, authz_vpc.id()) + .await + .map_err(SubnetError::External)?; Ok(( authz::VpcSubnet::new( authz_vpc.clone(), @@ -888,6 +882,12 @@ impl DataStore { "deletion failed due to concurrent modification", )); } else { + self.vpc_system_router_ensure_subnet_routes( + opctx, + db_subnet.vpc_id, + ) + .await?; + Ok(()) } } @@ -901,7 +901,7 @@ impl DataStore { opctx.authorize(authz::Action::Modify, authz_subnet).await?; use db::schema::vpc_subnet::dsl; - diesel::update(dsl::vpc_subnet) + let out = diesel::update(dsl::vpc_subnet) .filter(dsl::time_deleted.is_null()) .filter(dsl::id.eq(authz_subnet.id())) .set(updates) @@ -913,7 +913,11 @@ impl DataStore { e, ErrorHandler::NotFoundByResource(authz_subnet), ) - }) + })?; + + self.vpc_system_router_ensure_subnet_routes(opctx, out.vpc_id).await?; + + Ok(out) } pub async fn subnet_list_instance_network_interfaces( @@ -1097,6 +1101,17 @@ impl DataStore { assert_eq!(authz_router.id(), route.vpc_router_id); opctx.authorize(authz::Action::CreateChild, authz_router).await?; + Self::router_create_route_on_connection( + route, + &*self.pool_connection_authorized(opctx).await?, + ) + .await + } + + pub async fn router_create_route_on_connection( + route: RouterRoute, + conn: &async_bb8_diesel::Connection, + ) -> CreateResult { use db::schema::router_route::dsl; let router_id = route.vpc_router_id; let name = route.name().clone(); @@ -1105,9 +1120,7 @@ impl DataStore { router_id, diesel::insert_into(dsl::router_route).values(route), ) - .insert_and_get_result_async( - &*self.pool_connection_authorized(opctx).await?, - ) + .insert_and_get_result_async(conn) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { @@ -1259,6 +1272,116 @@ impl DataStore { ) }) } + + /// Ensure the system router for a VPC has the correct set of subnet + /// routing rules, after any changes to a subnet. + pub async fn vpc_system_router_ensure_subnet_routes( + &self, + opctx: &OpContext, + vpc_id: Uuid, + ) -> Result<(), Error> { + // These rules are immutable from a user's perspective, and + // aren't something which they can meaningfully interact with, + // so uuid stability on e.g. VPC rename is not a primary concern. + // We make sure only to alter VPC subnet rules here: users may + // modify other system routes like internet gateways. + let conn = self.pool_connection_authorized(opctx).await?; + let log = opctx.log.clone(); + self.transaction_retry_wrapper("vpc_subnet_route_reconcile") + .transaction(&conn, |conn| { + let log = log.clone(); + async move { + use db::schema::router_route::dsl; + use db::schema::vpc_subnet::dsl as subnet; + use db::schema::vpc::dsl as vpc; + + let system_router_id = vpc::vpc + .filter(vpc::id.eq(vpc_id)) + .filter(vpc::time_deleted.is_null()) + .select(vpc::system_router_id) + .limit(1) + .get_result_async(&conn) + .await?; + + let valid_subnets: Vec = subnet::vpc_subnet + .filter(subnet::id.eq(vpc_id)) + .filter(subnet::time_deleted.is_null()) + .select(VpcSubnet::as_select()) + .load_async(&conn) + .await?; + + let current_rules: Vec = dsl::router_route + .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::vpc_router_id.eq(system_router_id)) + .select(RouterRoute::as_select()) + .load_async(&conn) + .await?; + + // Build the add/delete sets. + let expected_names: HashSet = valid_subnets.iter() + .map(|v| v.identity.name.clone()) + .collect(); + + let mut found_names = HashSet::new(); + let mut invalid = Vec::new(); + for rule in current_rules { + let id = rule.id(); + match (rule.kind.0, rule.target.0) { + (ExternalRouteKind::VpcSubnet, RouteTarget::Subnet(n)) + if expected_names.contains(Name::ref_cast(&n)) => + {let _ = found_names.insert(n.into());}, + _ => invalid.push(id), + } + } + + // Add/Remove routes. Retry if numebr is incorrect due to + // concurrent modification. + let now = Utc::now(); + let to_update = invalid.len(); + let updated_rows = diesel::update(dsl::router_route) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq_any(invalid)) + .set(dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await?; + + if updated_rows != to_update { + return Err(DieselError::RollbackTransaction); + } + + // Duplicate rules are caught here using the UNIQUE constraint + // on names in a router. Only nexus can alter the system router, + // so there is no risk of collision with user-specified names. + for subnet in expected_names.difference(&found_names) { + let route_id = Uuid::new_v4(); + // XXX this is fallible as it is based on subnet name. + // need to control this somewhere sane. + let Ok(route) = db::model::RouterRoute::for_subnet( + route_id, + system_router_id, + subnet.clone(), + ) else { + error!( + log, + "Reconciling VPC routes: name {} in vpc {} is too long", + subnet, + vpc_id, + ); + continue; + }; + + match Self::router_create_route_on_connection(route, &conn).await { + Err(Error::Conflict { .. }) => return Err(DieselError::RollbackTransaction), + Err(_) => return Err(DieselError::NotFound), + _ => {}, + } + } + + Ok(()) + }}).await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } } #[cfg(test)] diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index 647ee05da5..475ff0bf58 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -246,7 +246,8 @@ async fn svc_create_v4_route( let default_route_id = sagactx.lookup::("default_v4_route_id")?; let default_route = "0.0.0.0/0".parse().expect("known-valid specifier for a default route"); - svc_create_route(sagactx, default_route_id, default_route).await + svc_create_route(sagactx, default_route_id, default_route, "default_v4") + .await } async fn svc_create_v4_route_undo( @@ -262,7 +263,8 @@ async fn svc_create_v6_route( let default_route_id = sagactx.lookup::("default_v6_route_id")?; let default_route = "::/0".parse().expect("known-valid specifier for a default route"); - svc_create_route(sagactx, default_route_id, default_route).await + svc_create_route(sagactx, default_route_id, default_route, "default_v6") + .await } async fn svc_create_v6_route_undo( @@ -276,6 +278,7 @@ async fn svc_create_route( sagactx: NexusActionContext, route_id: Uuid, default_net: IpNet, + name: &str, ) -> Result<(), ActionError> { let osagactx = sagactx.user_data(); let params = sagactx.saga_params::()?; @@ -292,7 +295,7 @@ async fn svc_create_route( RouterRouteKind::Default, params::RouterRouteCreate { identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), + name: name.parse().unwrap(), description: "The default route of a vpc".to_string(), }, target: RouteTarget::InternetGateway("outbound".parse().unwrap()), @@ -436,20 +439,15 @@ async fn svc_create_subnet_route( let system_router_id = sagactx.lookup::("system_router_id")?; let authz_router = sagactx.lookup::("router")?; let route_id = sagactx.lookup::("default_subnet_route_id")?; + let (_, db_subnet) = + sagactx.lookup::<(authz::VpcSubnet, db::model::VpcSubnet)>("subnet")?; - let route = db::model::RouterRoute::new( + let route = db::model::RouterRoute::for_subnet( route_id, system_router_id, - RouterRouteKind::Default, - params::RouterRouteCreate { - identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), - description: "The default route of a vpc".to_string(), - }, - target: RouteTarget::Subnet("default".parse().unwrap()), - destination: RouteDestination::Subnet("default".parse().unwrap()), - }, - ); + db_subnet.identity.name, + ) + .expect("default subnet name is short enough for route naming"); osagactx .datastore() From 22b71bb3c7a598f441e30aca2d4c30b4e29bdae7 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 15 May 2024 16:57:32 +0100 Subject: [PATCH 06/59] We now have valid, sane, default system routes These update in response to VPC subnet changes. Now to plumb them into OPTE. --- nexus/db-model/src/vpc_route.rs | 2 +- nexus/db-queries/src/db/datastore/vpc.rs | 31 ++++++++++-- nexus/src/app/sagas/vpc_create.rs | 64 +----------------------- nexus/src/app/vpc_router.rs | 1 - 4 files changed, 29 insertions(+), 69 deletions(-) diff --git a/nexus/db-model/src/vpc_route.rs b/nexus/db-model/src/vpc_route.rs index 2c561325a5..dda7f0b785 100644 --- a/nexus/db-model/src/vpc_route.rs +++ b/nexus/db-model/src/vpc_route.rs @@ -133,7 +133,7 @@ impl RouterRoute { system_router_id: Uuid, subnet: Name, ) -> Result { - let name = format!("subnet_{}", subnet).parse().map_err(|_| ())?; + let name = format!("sn-{}", subnet).parse().map_err(|_| ())?; Ok(Self::new( route_id, system_router_id, diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 029f46e3c1..d200f67663 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -140,18 +140,26 @@ impl DataStore { // Unwrap safety: these are known valid CIDR blocks. let default_ips = [ - ("0.0.0.0/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V4_ROUTE_ID), - ("::/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V6_ROUTE_ID), + ( + "default-v4", + "0.0.0.0/0".parse().unwrap(), + *SERVICES_VPC_DEFAULT_V4_ROUTE_ID, + ), + ( + "default-v6", + "::/0".parse().unwrap(), + *SERVICES_VPC_DEFAULT_V6_ROUTE_ID, + ), ]; - for (default, uuid) in default_ips { + for (name, default, uuid) in default_ips { let route = RouterRoute::new( uuid, SERVICES_VPC.system_router_id, ExternalRouteKind::Default, nexus_types::external_api::params::RouterRouteCreate { identity: IdentityMetadataCreateParams { - name: "default".parse().unwrap(), + name: name.parse().unwrap(), description: "Default internet gateway route for Oxide Services" .to_string(), @@ -1036,6 +1044,18 @@ impl DataStore { ErrorHandler::NotFoundByResource(authz_router), ) })?; + + // All child routes are deleted. + use db::schema::router_route::dsl as rr; + let now = Utc::now(); + diesel::update(rr::router_route) + .filter(rr::time_deleted.is_null()) + .filter(rr::vpc_router_id.eq(authz_router.id())) + .set(rr::time_deleted.eq(now)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) } @@ -1291,6 +1311,7 @@ impl DataStore { .transaction(&conn, |conn| { let log = log.clone(); async move { + use db::schema::router_route::dsl; use db::schema::vpc_subnet::dsl as subnet; use db::schema::vpc::dsl as vpc; @@ -1304,7 +1325,7 @@ impl DataStore { .await?; let valid_subnets: Vec = subnet::vpc_subnet - .filter(subnet::id.eq(vpc_id)) + .filter(subnet::vpc_id.eq(vpc_id)) .filter(subnet::time_deleted.is_null()) .select(VpcSubnet::as_select()) .load_async(&conn) diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index 475ff0bf58..cc62d9315d 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -57,10 +57,6 @@ declare_saga_actions! { + svc_create_subnet - svc_create_subnet_undo } - VPC_CREATE_SUBNET_ROUTE -> "route" { - + svc_create_subnet_route - - svc_create_subnet_route_undo - } VPC_UPDATE_FIREWALL -> "firewall" { + svc_update_firewall - svc_update_firewall_undo @@ -97,11 +93,6 @@ pub fn create_dag( "GenerateDefaultV6RouteId", ACTION_GENERATE_ID.as_ref(), )); - builder.append(Node::action( - "default_subnet_route_id", - "GenerateDefaultV6RouteId", - ACTION_GENERATE_ID.as_ref(), - )); builder.append(Node::action( "default_subnet_id", "GenerateDefaultSubnetId", @@ -112,7 +103,6 @@ pub fn create_dag( builder.append(vpc_create_v4_route_action()); builder.append(vpc_create_v6_route_action()); builder.append(vpc_create_subnet_action()); - builder.append(vpc_create_subnet_route_action()); builder.append(vpc_update_firewall_action()); builder.append(vpc_notify_sleds_action()); @@ -246,7 +236,7 @@ async fn svc_create_v4_route( let default_route_id = sagactx.lookup::("default_v4_route_id")?; let default_route = "0.0.0.0/0".parse().expect("known-valid specifier for a default route"); - svc_create_route(sagactx, default_route_id, default_route, "default_v4") + svc_create_route(sagactx, default_route_id, default_route, "default-v4") .await } @@ -263,7 +253,7 @@ async fn svc_create_v6_route( let default_route_id = sagactx.lookup::("default_v6_route_id")?; let default_route = "::/0".parse().expect("known-valid specifier for a default route"); - svc_create_route(sagactx, default_route_id, default_route, "default_v6") + svc_create_route(sagactx, default_route_id, default_route, "default-v6") .await } @@ -427,56 +417,6 @@ async fn svc_create_subnet_undo( Ok(()) } -async fn svc_create_subnet_route( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let system_router_id = sagactx.lookup::("system_router_id")?; - let authz_router = sagactx.lookup::("router")?; - let route_id = sagactx.lookup::("default_subnet_route_id")?; - let (_, db_subnet) = - sagactx.lookup::<(authz::VpcSubnet, db::model::VpcSubnet)>("subnet")?; - - let route = db::model::RouterRoute::for_subnet( - route_id, - system_router_id, - db_subnet.identity.name, - ) - .expect("default subnet name is short enough for route naming"); - - osagactx - .datastore() - .router_create_route(&opctx, &authz_router, route) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - -async fn svc_create_subnet_route_undo( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - let authz_router = sagactx.lookup::("router")?; - let route_id = sagactx.lookup::("default_subnet_route_id")?; - let authz_route = authz::RouterRoute::new( - authz_router, - route_id, - LookupType::ById(route_id), - ); - osagactx.datastore().router_delete_route(&opctx, &authz_route).await?; - Ok(()) -} - async fn svc_update_firewall( sagactx: NexusActionContext, ) -> Result, ActionError> { diff --git a/nexus/src/app/vpc_router.rs b/nexus/src/app/vpc_router.rs index 523a450bbd..e65b2a8605 100644 --- a/nexus/src/app/vpc_router.rs +++ b/nexus/src/app/vpc_router.rs @@ -114,7 +114,6 @@ impl super::Nexus { .await } - // TODO: When a router is deleted all its routes should be deleted // TODO: When a router is deleted it should be unassociated w/ any subnets it may be associated with // or trigger an error pub(crate) async fn vpc_delete_router( From 160853b2cb92a012b0124557a16d9f7e76ea08dc Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 15 May 2024 17:42:28 +0100 Subject: [PATCH 07/59] Start refreshing API specs --- nexus/tests/output/nexus_tags.txt | 10 + openapi/nexus.json | 1695 ++++++++++++++++++++++++----- 2 files changed, 1413 insertions(+), 292 deletions(-) diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index a32fe5c4b9..35d8c32561 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -232,6 +232,16 @@ vpc_delete DELETE /v1/vpcs/{vpc} vpc_firewall_rules_update PUT /v1/vpc-firewall-rules vpc_firewall_rules_view GET /v1/vpc-firewall-rules vpc_list GET /v1/vpcs +vpc_router_create POST /v1/vpc-routers +vpc_router_delete DELETE /v1/vpc-routers/{router} +vpc_router_list GET /v1/vpc-routers +vpc_router_route_create POST /v1/vpc-router-routes +vpc_router_route_delete DELETE /v1/vpc-router-routes/{route} +vpc_router_route_list GET /v1/vpc-router-routes +vpc_router_route_update PUT /v1/vpc-router-routes/{route} +vpc_router_route_view GET /v1/vpc-router-routes/{route} +vpc_router_update PUT /v1/vpc-routers/{router} +vpc_router_view GET /v1/vpc-routers/{router} vpc_subnet_create POST /v1/vpc-subnets vpc_subnet_delete DELETE /v1/vpc-subnets/{subnet} vpc_subnet_list GET /v1/vpc-subnets diff --git a/openapi/nexus.json b/openapi/nexus.json index 2bf6f0a6ff..55f83f4a24 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -8346,13 +8346,14 @@ } } }, - "/v1/vpc-subnets": { + "/v1/vpc-router-routes": { "get": { "tags": [ "vpcs" ], - "summary": "List subnets", - "operationId": "vpc_subnet_list", + "summary": "List routes", + "description": "List the routes associated with a router in a particular VPC.", + "operationId": "vpc_router_route_list", "parameters": [ { "in": "query", @@ -8382,6 +8383,14 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "sort_by", @@ -8392,7 +8401,7 @@ { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8404,7 +8413,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetResultsPage" + "$ref": "#/components/schemas/RouterRouteResultsPage" } } } @@ -8418,7 +8427,7 @@ }, "x-dropshot-pagination": { "required": [ - "vpc" + "router" ] } }, @@ -8426,8 +8435,8 @@ "tags": [ "vpcs" ], - "summary": "Create subnet", - "operationId": "vpc_subnet_create", + "summary": "Create route", + "operationId": "vpc_router_route_create", "parameters": [ { "in": "query", @@ -8439,19 +8448,27 @@ }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetCreate" + "$ref": "#/components/schemas/RouterRouteCreate" } } }, @@ -8463,7 +8480,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8477,18 +8494,18 @@ } } }, - "/v1/vpc-subnets/{subnet}": { + "/v1/vpc-router-routes/{route}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch subnet", - "operationId": "vpc_subnet_view", + "summary": "Fetch route", + "operationId": "vpc_router_route_view", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8502,10 +8519,19 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8517,7 +8543,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8534,13 +8560,13 @@ "tags": [ "vpcs" ], - "summary": "Update subnet", - "operationId": "vpc_subnet_update", + "summary": "Update route", + "operationId": "vpc_router_route_update", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8554,10 +8580,18 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8567,7 +8601,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetUpdate" + "$ref": "#/components/schemas/RouterRouteUpdate" } } }, @@ -8579,7 +8613,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8596,13 +8630,13 @@ "tags": [ "vpcs" ], - "summary": "Delete subnet", - "operationId": "vpc_subnet_delete", + "summary": "Delete route", + "operationId": "vpc_router_route_delete", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8616,10 +8650,18 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8638,23 +8680,14 @@ } } }, - "/v1/vpc-subnets/{subnet}/network-interfaces": { + "/v1/vpc-routers": { "get": { "tags": [ "vpcs" ], - "summary": "List network interfaces", - "operationId": "vpc_subnet_list_network_interfaces", + "summary": "List routers", + "operationId": "vpc_router_list", "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "limit", @@ -8705,7 +8738,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" + "$ref": "#/components/schemas/VpcRouterResultsPage" } } } @@ -8718,89 +8751,30 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "vpc" + ] } - } - }, - "/v1/vpcs": { - "get": { + }, + "post": { "tags": [ "vpcs" ], - "summary": "List VPCs", - "operationId": "vpc_list", + "summary": "Create VPC router", + "operationId": "vpc_router_create", "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "project" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create VPC", - "operationId": "vpc_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8811,7 +8785,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcCreate" + "$ref": "#/components/schemas/VpcRouterCreate" } } }, @@ -8823,7 +8797,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -8837,18 +8811,18 @@ } } }, - "/v1/vpcs/{vpc}": { + "/v1/vpc-routers/{router}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch VPC", - "operationId": "vpc_view", + "summary": "Fetch router", + "operationId": "vpc_router_view", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8857,7 +8831,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8869,7 +8851,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -8886,13 +8868,13 @@ "tags": [ "vpcs" ], - "summary": "Update a VPC", - "operationId": "vpc_update", + "summary": "Update router", + "operationId": "vpc_router_update", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8901,7 +8883,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8911,7 +8901,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcUpdate" + "$ref": "#/components/schemas/VpcRouterUpdate" } } }, @@ -8923,7 +8913,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -8940,13 +8930,13 @@ "tags": [ "vpcs" ], - "summary": "Delete VPC", - "operationId": "vpc_delete", + "summary": "Delete router", + "operationId": "vpc_router_delete", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8955,7 +8945,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8973,43 +8971,671 @@ } } } - } - }, - "components": { - "schemas": { - "Address": { - "description": "An address tied to an address lot.", - "type": "object", - "properties": { - "address": { - "description": "The address and prefix length of this address.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] + }, + "/v1/vpc-subnets": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List subnets", + "operationId": "vpc_subnet_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } }, - "address_lot": { - "description": "The address lot this address is drawn from.", - "allOf": [ - { - "$ref": "#/components/schemas/NameOrId" - } - ] - } - }, - "required": [ - "address", - "address_lot" - ] - }, - "AddressConfig": { - "description": "A set of addresses associated with a port configuration.", - "type": "object", - "properties": { - "addresses": { - "description": "The set of addresses assigned to the port configuration.", - "type": "array", + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnetResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "vpc" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create subnet", + "operationId": "vpc_subnet_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnetCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpc-subnets/{subnet}": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "Fetch subnet", + "operationId": "vpc_subnet_view", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "vpcs" + ], + "summary": "Update subnet", + "operationId": "vpc_subnet_update", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnetUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "vpcs" + ], + "summary": "Delete subnet", + "operationId": "vpc_subnet_delete", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpc-subnets/{subnet}/network-interfaces": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List network interfaces", + "operationId": "vpc_subnet_list_network_interfaces", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/vpcs": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List VPCs", + "operationId": "vpc_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create VPC", + "operationId": "vpc_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpcs/{vpc}": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "Fetch VPC", + "operationId": "vpc_view", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "vpcs" + ], + "summary": "Update a VPC", + "operationId": "vpc_update", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "vpcs" + ], + "summary": "Delete VPC", + "operationId": "vpc_delete", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Address": { + "description": "An address tied to an address lot.", + "type": "object", + "properties": { + "address": { + "description": "The address and prefix length of this address.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "address_lot": { + "description": "The address lot this address is drawn from.", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + } + }, + "required": [ + "address", + "address_lot" + ] + }, + "AddressConfig": { + "description": "A set of addresses associated with a port configuration.", + "type": "object", + "properties": { + "addresses": { + "description": "The set of addresses assigned to the port configuration.", + "type": "array", "items": { "$ref": "#/components/schemas/Address" } @@ -15617,22 +16243,169 @@ } }, "required": [ - "description", - "id", - "name", - "time_created", - "time_modified" + "description", + "id", + "name", + "time_created", + "time_modified" + ] + }, + "ProjectCreate": { + "description": "Create-time parameters for a `Project`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "description", + "name" + ] + }, + "ProjectResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Project" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "ProjectRole": { + "type": "string", + "enum": [ + "admin", + "collaborator", + "viewer" + ] + }, + "ProjectRolePolicy": { + "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", + "type": "object", + "properties": { + "role_assignments": { + "description": "Roles directly assigned on this resource", + "type": "array", + "items": { + "$ref": "#/components/schemas/ProjectRoleRoleAssignment" + } + } + }, + "required": [ + "role_assignments" + ] + }, + "ProjectRoleRoleAssignment": { + "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", + "type": "object", + "properties": { + "identity_id": { + "type": "string", + "format": "uuid" + }, + "identity_type": { + "$ref": "#/components/schemas/IdentityType" + }, + "role_name": { + "$ref": "#/components/schemas/ProjectRole" + } + }, + "required": [ + "identity_id", + "identity_type", + "role_name" + ] + }, + "ProjectUpdate": { + "description": "Updateable properties of a `Project`", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, + "Rack": { + "description": "View of an Rack", + "type": "object", + "properties": { + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "id", + "time_created", + "time_modified" + ] + }, + "RackResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Rack" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" ] }, - "ProjectCreate": { - "description": "Create-time parameters for a `Project`", + "Role": { + "description": "View of a Role", "type": "object", "properties": { "description": { "type": "string" }, "name": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/RoleName" } }, "required": [ @@ -15640,7 +16413,14 @@ "name" ] }, - "ProjectResultsPage": { + "RoleName": { + "title": "A name for a built-in role", + "description": "Role names consist of two string components separated by dot (\".\").", + "type": "string", + "pattern": "[a-z-]+\\.[a-z-]+", + "maxLength": 63 + }, + "RoleResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -15648,7 +16428,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/Project" + "$ref": "#/components/schemas/Role" } }, "next_page": { @@ -15661,77 +16441,284 @@ "items" ] }, - "ProjectRole": { - "type": "string", - "enum": [ - "admin", - "collaborator", - "viewer" + "Route": { + "description": "A route to a destination network through a gateway address.", + "type": "object", + "properties": { + "dst": { + "description": "The route destination.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "gw": { + "description": "The route gateway.", + "type": "string", + "format": "ip" + }, + "vid": { + "nullable": true, + "description": "VLAN id the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "dst", + "gw" ] }, - "ProjectRolePolicy": { - "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", + "RouteConfig": { + "description": "Route configuration data associated with a switch port configuration.", "type": "object", "properties": { - "role_assignments": { - "description": "Roles directly assigned on this resource", + "routes": { + "description": "The set of routes assigned to a switch port.", "type": "array", "items": { - "$ref": "#/components/schemas/ProjectRoleRoleAssignment" + "$ref": "#/components/schemas/Route" } } }, "required": [ - "role_assignments" + "routes" ] }, - "ProjectRoleRoleAssignment": { - "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", - "type": "object", - "properties": { - "identity_id": { - "type": "string", - "format": "uuid" + "RouteDestination": { + "description": "A `RouteDestination` is used to match traffic with a routing rule, on the destination of that traffic.\n\nWhen traffic is to be sent to a destination that is within a given `RouteDestination`, the corresponding `RouterRoute` applies, and traffic will be forward to the `RouteTarget` for that rule.", + "oneOf": [ + { + "description": "Route applies to traffic destined for a specific IP address", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] }, - "identity_type": { - "$ref": "#/components/schemas/IdentityType" + { + "description": "Route applies to traffic destined for a specific IP subnet", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip_net" + ] + }, + "value": { + "$ref": "#/components/schemas/IpNet" + } + }, + "required": [ + "type", + "value" + ] }, - "role_name": { - "$ref": "#/components/schemas/ProjectRole" + { + "description": "Route applies to traffic destined for the given VPC.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Route applies to traffic", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "RouteTarget": { + "description": "A `RouteTarget` describes the possible locations that traffic matching a route destination can be sent.", + "oneOf": [ + { + "description": "Forward traffic to a particular IP address.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to a VPC", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to a VPC Subnet", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to a specific instance", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "instance" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to an internet gateway", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internet_gateway" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Drop matching traffic", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "drop" + ] + } + }, + "required": [ + "type" + ] } - }, - "required": [ - "identity_id", - "identity_type", - "role_name" ] }, - "ProjectUpdate": { - "description": "Updateable properties of a `Project`", + "RouterRoute": { + "description": "A route defines a rule that governs where traffic should be sent based on its destination.", "type": "object", "properties": { "description": { - "nullable": true, + "description": "human-readable free-form text about a resource", "type": "string" }, + "destination": { + "$ref": "#/components/schemas/RouteDestination" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "kind": { + "description": "Describes the kind of router. Set at creation. `read-only`", + "allOf": [ + { + "$ref": "#/components/schemas/RouterRouteKind" + } + ] + }, "name": { - "nullable": true, + "description": "unique, mutable, user-controlled identifier for each resource", "allOf": [ { "$ref": "#/components/schemas/Name" } ] - } - } - }, - "Rack": { - "description": "View of an Rack", - "type": "object", - "properties": { - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" + }, + "target": { + "$ref": "#/components/schemas/RouteTarget" }, "time_created": { "description": "timestamp when this resource was created", @@ -15742,59 +16729,83 @@ "description": "timestamp when this resource was last modified", "type": "string", "format": "date-time" + }, + "vpc_router_id": { + "description": "The ID of the VPC Router to which the route belongs", + "type": "string", + "format": "uuid" } }, "required": [ + "description", + "destination", "id", + "kind", + "name", + "target", "time_created", - "time_modified" - ] - }, - "RackResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/Rack" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" + "time_modified", + "vpc_router_id" ] }, - "Role": { - "description": "View of a Role", + "RouterRouteCreate": { + "description": "Create-time parameters for a `RouterRoute`", "type": "object", "properties": { "description": { "type": "string" }, + "destination": { + "$ref": "#/components/schemas/RouteDestination" + }, "name": { - "$ref": "#/components/schemas/RoleName" + "$ref": "#/components/schemas/Name" + }, + "target": { + "$ref": "#/components/schemas/RouteTarget" } }, "required": [ "description", - "name" + "destination", + "name", + "target" ] }, - "RoleName": { - "title": "A name for a built-in role", - "description": "Role names consist of two string components separated by dot (\".\").", - "type": "string", - "pattern": "[a-z-]+\\.[a-z-]+", - "maxLength": 63 + "RouterRouteKind": { + "description": "The kind of a `RouterRoute`\n\nThe kind determines certain attributes such as if the route is modifiable and describes how or where the route was created.", + "oneOf": [ + { + "description": "Determines the default destination of traffic, such as whether it goes to the internet or not.\n\n`Destination: An Internet Gateway` `Modifiable: true`", + "type": "string", + "enum": [ + "default" + ] + }, + { + "description": "Automatically added for each VPC Subnet in the VPC\n\n`Destination: A VPC Subnet` `Modifiable: false`", + "type": "string", + "enum": [ + "vpc_subnet" + ] + }, + { + "description": "Automatically added when VPC peering is established\n\n`Destination: A different VPC` `Modifiable: false`", + "type": "string", + "enum": [ + "vpc_peering" + ] + }, + { + "description": "Created by a user; see `RouteTarget`\n\n`Destination: User defined` `Modifiable: true`", + "type": "string", + "enum": [ + "custom" + ] + } + ] }, - "RoleResultsPage": { + "RouterRouteResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -15802,7 +16813,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/Role" + "$ref": "#/components/schemas/RouterRoute" } }, "next_page": { @@ -15815,50 +16826,32 @@ "items" ] }, - "Route": { - "description": "A route to a destination network through a gateway address.", + "RouterRouteUpdate": { + "description": "Updateable properties of a `RouterRoute`", "type": "object", "properties": { - "dst": { - "description": "The route destination.", + "description": { + "nullable": true, + "type": "string" + }, + "destination": { + "$ref": "#/components/schemas/RouteDestination" + }, + "name": { + "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/IpNet" + "$ref": "#/components/schemas/Name" } ] }, - "gw": { - "description": "The route gateway.", - "type": "string", - "format": "ip" - }, - "vid": { - "nullable": true, - "description": "VLAN id the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "dst", - "gw" - ] - }, - "RouteConfig": { - "description": "Route configuration data associated with a switch port configuration.", - "type": "object", - "properties": { - "routes": { - "description": "The set of routes assigned to a switch port.", - "type": "array", - "items": { - "$ref": "#/components/schemas/Route" - } + "target": { + "$ref": "#/components/schemas/RouteTarget" } }, "required": [ - "routes" + "destination", + "target" ] }, "SamlIdentityProvider": { @@ -18956,10 +19949,128 @@ "items" ] }, + "VpcRouter": { + "description": "A VPC router defines a series of rules that indicate where traffic should be sent depending on its destination.", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "kind": { + "$ref": "#/components/schemas/VpcRouterKind" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + }, + "vpc_id": { + "description": "The VPC to which the router belongs.", + "type": "string", + "format": "uuid" + } + }, + "required": [ + "description", + "id", + "kind", + "name", + "time_created", + "time_modified", + "vpc_id" + ] + }, + "VpcRouterCreate": { + "description": "Create-time parameters for a `VpcRouter`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "description", + "name" + ] + }, + "VpcRouterKind": { + "type": "string", + "enum": [ + "system", + "custom" + ] + }, + "VpcRouterResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/VpcRouter" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "VpcRouterUpdate": { + "description": "Updateable properties of a `VpcRouter`", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, "VpcSubnet": { - "description": "A VPC subnet represents a logical grouping for instances that allows network traffic between them, within a IPv4 subnetwork or optionall an IPv6 subnetwork.", + "description": "A VPC subnet represents a logical grouping for instances that allows network traffic between them, within a IPv4 subnetwork or optionally an IPv6 subnetwork.", "type": "object", "properties": { + "custom_router_id": { + "nullable": true, + "description": "ID for an attached custom router.", + "type": "string", + "format": "uuid" + }, "description": { "description": "human-readable free-form text about a resource", "type": "string" From 034dd0fe8424bb8325b2623d45639649162763f9 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 16 May 2024 15:21:41 +0100 Subject: [PATCH 08/59] Iterating. --- common/src/api/internal/shared.rs | 41 +++++- illumos-utils/src/opte/port_manager.rs | 49 +++++++ nexus/src/app/background/init.rs | 20 ++- nexus/src/app/background/mod.rs | 1 + nexus/src/app/background/vpc_routes.rs | 98 +++++++++++++ openapi/sled-agent.json | 183 +++++++++++++++++++++++++ sled-agent/src/http_entrypoints.rs | 30 +++- sled-agent/src/sled_agent.rs | 10 +- 8 files changed, 427 insertions(+), 5 deletions(-) create mode 100644 nexus/src/app/background/vpc_routes.rs diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 9d9ff083e4..96f0fecd95 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -6,13 +6,13 @@ use crate::{ address::NUM_SOURCE_NAT_PORTS, - api::external::{self, BfdMode, ImportExportPolicy, IpNet, Name}, + api::external::{self, BfdMode, ImportExportPolicy, IpNet, Name, Vni}, }; use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr}, str::FromStr, @@ -590,6 +590,43 @@ impl TryFrom<&[IpNetwork]> for IpAllowList { } } +/// A VPC route resolved into a concrete target. +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +pub struct ReifiedVpcRoute { + pub dest: IpNet, + pub target: RouterTarget, +} + +/// The target for a given router entry. +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +#[serde(tag = "type", rename_all = "snake_case", content = "value")] +pub enum RouterTarget { + Drop, + InternetGateway, + Ip(IpAddr), + VpcSubnet(IpNet), +} + +/// XX +#[derive( + Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +pub struct RouterId { + pub vni: Vni, + pub subnet: Option, +} + +/// XX +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +pub struct ReifiedVpcRouteSet { + pub id: RouterId, + pub routes: HashSet, +} + #[cfg(test)] mod tests { use crate::api::{ diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index d6b4963607..48d6bb7fcb 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -16,6 +16,10 @@ use ipnetwork::IpNetwork; use omicron_common::api::external; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; +use omicron_common::api::internal::shared::ReifiedVpcRoute; +use omicron_common::api::internal::shared::ReifiedVpcRouteSet; +use omicron_common::api::internal::shared::RouterId; +use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget; use omicron_common::api::internal::shared::SourceNatConfig; use oxide_vpc::api::AddRouterEntryReq; use oxide_vpc::api::DhcpCfg; @@ -36,6 +40,8 @@ use slog::error; use slog::info; use slog::Logger; use std::collections::BTreeMap; +use std::collections::HashMap; +use std::collections::HashSet; use std::net::IpAddr; use std::net::Ipv6Addr; use std::sync::atomic::AtomicU64; @@ -60,6 +66,13 @@ struct PortManagerInner { // Map of all ports, keyed on the interface Uuid and its kind // (which includes the Uuid of the parent instance or service) ports: Mutex>, + + // XX: vs. Hashmap? + // XX: Should this be the UUID of the VPC? The rulesets are + // arguably shared v4+v6, although today we don't yet + // allow dual-stack, let alone v6. + // Map of all current resolved routes + routes: Mutex>>, } impl PortManagerInner { @@ -86,6 +99,7 @@ impl PortManager { next_port_id: AtomicU64::new(0), underlay_ip, ports: Mutex::new(BTreeMap::new()), + routes: Mutex::new(Default::default()), }); Self { inner } @@ -400,6 +414,24 @@ impl PortManager { "route" => ?route, ); + // XX: this is probably not the right initialisation here... + // XX: VPC rules should probably come from ctl plane. + let mut routes = self.inner.routes.lock().unwrap(); + routes.entry(RouterId { vni: nic.vni, subnet: None }).or_insert_with( + || { + let mut out = HashSet::new(); + out.insert(ReifiedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway, + }); + out.insert(ReifiedVpcRoute { + dest: "::/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway, + }); + out + }, + ); + info!( self.inner.log, "Created OPTE port"; @@ -408,6 +440,23 @@ impl PortManager { Ok((port, ticket)) } + pub fn vpc_routes_list(&self) -> Vec { + let routes = self.inner.routes.lock().unwrap(); + routes + .iter() + .map(|(k, v)| ReifiedVpcRouteSet { id: *k, routes: v.clone() }) + .collect() + } + + pub fn vpc_routes_ensure(&self, new_routes: Vec) { + let mut routes = self.inner.routes.lock().unwrap(); + // *routes = new_routes; + drop(routes); + + // XX: compute deltas. + // XX: push down to OPTE. + } + /// Ensure external IPs for an OPTE port are up to date. #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] pub fn external_ips_ensure( diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index d2f940018d..6210b0dd3e 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -22,6 +22,7 @@ use super::region_replacement; use super::service_firewall_rules; use super::sync_service_zone_nat::ServiceZoneNatTracker; use super::sync_switch_configuration::SwitchPortSettingsManager; +use super::vpc_routes; use crate::app::oximeter::PRODUCER_LEASE_DURATION; use crate::app::sagas::SagaRequest; use nexus_config::BackgroundTaskConfig; @@ -100,6 +101,9 @@ pub struct BackgroundTasks { /// task handle for propagation of VPC firewall rules for Omicron services /// with external network connectivity, pub task_service_firewall_propagation: common::TaskHandle, + + /// task handle for propagation of VPC router rules to all OPTE ports + pub task_vpc_route_manager: common::TaskHandle, } impl BackgroundTasks { @@ -368,6 +372,7 @@ impl BackgroundTasks { vec![], ) }; + // Background task: service firewall rule propagation let task_service_firewall_propagation = driver.register( String::from("service_firewall_rule_propagation"), @@ -377,12 +382,24 @@ impl BackgroundTasks { ), config.service_firewall_propagation.period_secs, Box::new(service_firewall_rules::ServiceRulePropagator::new( - datastore, + datastore.clone(), )), opctx.child(BTreeMap::new()), vec![], ); + let task_vpc_route_manager = { + let watcher = vpc_routes::VpcRouteManager::new(datastore); + driver.register( + "vpc_route_manager".to_string(), + "propagates updated VPC routes to all OPTE ports".into(), + config.switch_port_settings_manager.period_secs, + Box::new(watcher), + opctx.child(BTreeMap::new()), + vec![], + ) + }; + BackgroundTasks { driver, task_internal_dns_config, @@ -404,6 +421,7 @@ impl BackgroundTasks { task_region_replacement, task_instance_watcher, task_service_firewall_propagation, + task_vpc_route_manager, } } diff --git a/nexus/src/app/background/mod.rs b/nexus/src/app/background/mod.rs index 512c782b2e..db1d23f221 100644 --- a/nexus/src/app/background/mod.rs +++ b/nexus/src/app/background/mod.rs @@ -25,5 +25,6 @@ mod service_firewall_rules; mod status; mod sync_service_zone_nat; mod sync_switch_configuration; +mod vpc_routes; pub use init::BackgroundTasks; diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs new file mode 100644 index 0000000000..c5c1aaf104 --- /dev/null +++ b/nexus/src/app/background/vpc_routes.rs @@ -0,0 +1,98 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Background task for propagating VPC routes (system and custom) to sleds. + +use super::common::BackgroundTask; +use futures::future::BoxFuture; +use futures::FutureExt; +use nexus_db_model::{Sled, SledState}; +use nexus_db_queries::{context::OpContext, db::DataStore}; +use nexus_networking::sled_client_from_address; +use nexus_types::{ + deployment::SledFilter, external_api::views::SledPolicy, identity::Asset, +}; +use omicron_common::api::{external::Vni, internal::shared::RouterId}; +use serde_json::json; +use sled_agent_client::types::ReifiedVpcRoute; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; + +pub struct VpcRouteManager { + datastore: Arc, +} + +impl VpcRouteManager { + pub fn new(datastore: Arc) -> Self { + Self { datastore } + } +} + +impl BackgroundTask for VpcRouteManager { + fn activate<'a>( + &'a mut self, + opctx: &'a OpContext, + ) -> BoxFuture<'a, serde_json::Value> { + async { + let log = &opctx.log; + + // XX: copied from omicron#5566 + let sleds = match self + .datastore + .sled_list_all_batched(opctx, SledFilter::InService) + .await + { + Ok(v) => v, + Err(e) => { + let msg = format!("failed to enumerate sleds: {:#}", e); + error!(&log, "{msg}"); + return json!({"error": msg}); + } + } + .into_iter() + .filter(|sled| { + matches!(sled.state(), SledState::Active) + && matches!(sled.policy(), SledPolicy::InService { .. }) + }); + + // Map sled db records to sled-agent clients + let sled_clients: Vec<(Sled, sled_agent_client::Client)> = sleds + .map(|sled| { + let client = sled_client_from_address( + sled.id(), + sled.address(), + &log, + ); + (sled, client) + }) + .collect(); + + // XX: actually reify rules. + let mut known_rules: HashMap> = + HashMap::new(); + + for (sled, client) in sled_clients { + let Ok(a) = client.list_vpc_routes().await else { + warn!( + log, + "failed to fetch current VPC route state from sled"; + "sled" => sled.serial_number(), + ); + continue; + }; + + // XX: Who decides what we want? Do we figure out the NICs + // here? Or take the sled at their word for what subnets + // they want? + + todo!() + } + + json!({}) + } + .boxed() + } +} diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 5da2b5c797..cf55e8ef74 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -953,6 +953,63 @@ } } }, + "/vpc-routes": { + "get": { + "summary": "Get the current state of VPC routing rules.", + "operationId": "list_vpc_routes", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ReifiedVpcRouteSet", + "type": "array", + "items": { + "$ref": "#/components/schemas/ReifiedVpcRouteSet" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "summary": "Update VPC routing rules.", + "operationId": "set_vpc_routes", + "requestBody": { + "content": { + "application/json": { + "schema": { + "title": "Array_of_ReifiedVpcRouteSet", + "type": "array", + "items": { + "$ref": "#/components/schemas/ReifiedVpcRouteSet" + } + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/zones": { "get": { "summary": "List the zones that are currently managed by the sled agent.", @@ -4252,6 +4309,42 @@ "rack_subnet" ] }, + "ReifiedVpcRoute": { + "description": "A VPC route resolved into a concrete target.", + "type": "object", + "properties": { + "dest": { + "$ref": "#/components/schemas/IpNet" + }, + "target": { + "$ref": "#/components/schemas/RouterTarget" + } + }, + "required": [ + "dest", + "target" + ] + }, + "ReifiedVpcRouteSet": { + "description": "XX", + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/RouterId" + }, + "routes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ReifiedVpcRoute" + }, + "uniqueItems": true + } + }, + "required": [ + "id", + "routes" + ] + }, "RouteConfig": { "type": "object", "properties": { @@ -4281,6 +4374,96 @@ "nexthop" ] }, + "RouterId": { + "description": "XX", + "type": "object", + "properties": { + "subnet": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "vni": { + "$ref": "#/components/schemas/Vni" + } + }, + "required": [ + "vni" + ] + }, + "RouterTarget": { + "description": "The target for a given router entry.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "drop" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internet_gateway" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc_subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/IpNet" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 99c7725fe3..f41cd5e817 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -33,7 +33,7 @@ use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::{ DiskRuntimeState, SledInstanceState, UpdateArtifactId, }; -use omicron_common::api::internal::shared::SwitchPorts; +use omicron_common::api::internal::shared::{ReifiedVpcRouteSet, SwitchPorts}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_hardware::DiskVariant; @@ -86,6 +86,8 @@ pub fn api() -> SledApiDescription { api.register(host_os_write_status_delete)?; api.register(inventory)?; api.register(bootstore_status)?; + api.register(list_vpc_routes)?; + api.register(set_vpc_routes)?; Ok(()) } @@ -1018,3 +1020,29 @@ async fn bootstore_status( .into(); Ok(HttpResponseOk(status)) } + +/// Get the current state of VPC routing rules. +#[endpoint { + method = GET, + path = "/vpc-routes", +}] +async fn list_vpc_routes( + request_context: RequestContext, +) -> Result>, HttpError> { + let sa = request_context.context(); + Ok(HttpResponseOk(sa.list_vpc_routes())) +} + +/// Update VPC routing rules. +#[endpoint { + method = PUT, + path = "/vpc-routes", +}] +async fn set_vpc_routes( + request_context: RequestContext, + body: TypedBody>, +) -> Result { + let sa = request_context.context(); + sa.set_vpc_routes(body.into_inner()); + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 39a5647420..01a6e3e9d7 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -51,7 +51,7 @@ use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; use omicron_common::api::internal::shared::{ - HostPortConfig, RackNetworkConfig, + HostPortConfig, RackNetworkConfig, ReifiedVpcRouteSet, }; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, @@ -1091,6 +1091,14 @@ impl SledAgent { self.inner.bootstore.clone() } + pub fn list_vpc_routes(&self) -> Vec { + self.inner.port_manager.vpc_routes_list() + } + + pub fn set_vpc_routes(&self, routes: Vec) { + self.inner.port_manager.vpc_routes_ensure(routes) + } + /// Return the metric producer registry. pub fn metrics_registry(&self) -> &ProducerRegistry { self.inner.metrics_manager.registry() From 87d9b26ee314736f11d7410c44fcd84adf87da09 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 17 May 2024 13:14:58 +0100 Subject: [PATCH 09/59] Route resolution (but not actual installation) --- clients/sled-agent-client/src/lib.rs | 5 + illumos-utils/src/opte/port_manager.rs | 9 + nexus/db-queries/src/db/datastore/instance.rs | 52 ++++ .../src/db/datastore/network_interface.rs | 22 ++ nexus/db-queries/src/db/datastore/vpc.rs | 280 ++++++++++++++++++ nexus/src/app/background/vpc_routes.rs | 153 +++++++++- schema/crdb/vpc-subnet-routing/up02.sql | 0 7 files changed, 512 insertions(+), 9 deletions(-) create mode 100644 schema/crdb/vpc-subnet-routing/up02.sql diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index a0145af910..58a1fc0d14 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -36,6 +36,7 @@ progenitor::generate_api!( RouteConfig = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, IpNet = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] } + RouterId = { derives = [PartialEq, Eq, Hash, Debug, Deserialize, Serialize] }, }, //TODO trade the manual transformations later in this file for the // replace directives below? @@ -51,6 +52,10 @@ progenitor::generate_api!( IpNetwork = ipnetwork::IpNetwork, PortFec = omicron_common::api::internal::shared::PortFec, PortSpeed = omicron_common::api::internal::shared::PortSpeed, + RouterId = omicron_common::api::internal::shared::RouterId, + ReifiedVpcRoute = omicron_common::api::internal::shared::ReifiedVpcRoute, + ReifiedVpcRouteSet = omicron_common::api::internal::shared::ReifiedVpcRouteSet, + RouterTarget = omicron_common::api::internal::shared::RouterTarget, SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig, Vni = omicron_common::api::external::Vni, NetworkInterface = omicron_common::api::internal::shared::NetworkInterface, diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 48d6bb7fcb..02b85e37bd 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -416,6 +416,7 @@ impl PortManager { // XX: this is probably not the right initialisation here... // XX: VPC rules should probably come from ctl plane. + // XX: need to delete safely after. let mut routes = self.inner.routes.lock().unwrap(); routes.entry(RouterId { vni: nic.vni, subnet: None }).or_insert_with( || { @@ -431,6 +432,9 @@ impl PortManager { out }, ); + routes + .entry(RouterId { vni: nic.vni, subnet: Some(nic.subnet) }) + .or_insert_with(|| HashSet::default()); info!( self.inner.log, @@ -450,6 +454,11 @@ impl PortManager { pub fn vpc_routes_ensure(&self, new_routes: Vec) { let mut routes = self.inner.routes.lock().unwrap(); + error!( + self.inner.log, + "I got routes I don't know what to do with!"; + "route_set" => format!("{new_routes:?}") + ); // *routes = new_routes; drop(routes); diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index ce40e20501..cd12cb6793 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -441,6 +441,58 @@ impl DataStore { Ok(result) } + /// Lists all instances on in-service sleds with active Propolis VMM + /// processes, returning the instance along with the VMM on which it's + /// running, the sled on which the VMM is running, and the project that owns + /// the instance. + /// + /// The query performed by this function is paginated by the sled's UUID. + pub async fn instance_and_vpc_list_by_sled_agent( + &self, + opctx: &OpContext, + pagparams: &DataPageParams<'_, Uuid>, + ) -> ListResultVec<(Sled, Instance, Vmm, Project)> { + use crate::db::schema::{ + instance::dsl as instance_dsl, project::dsl as project_dsl, + sled::dsl as sled_dsl, vmm::dsl as vmm_dsl, + }; + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + let result = paginated(sled_dsl::sled, sled_dsl::id, pagparams) + .filter(sled_dsl::time_deleted.is_null()) + .sled_filter(SledFilter::InService) + .inner_join( + vmm_dsl::vmm + .on(vmm_dsl::sled_id + .eq(sled_dsl::id) + .and(vmm_dsl::time_deleted.is_null())) + .inner_join( + instance_dsl::instance + .on(instance_dsl::id + .eq(vmm_dsl::instance_id) + .and(instance_dsl::time_deleted.is_null())) + .inner_join( + project_dsl::project.on(project_dsl::id + .eq(instance_dsl::project_id) + .and(project_dsl::time_deleted.is_null())), + ), + ), + ) + .sled_filter(SledFilter::InService) + .select(( + Sled::as_select(), + Instance::as_select(), + Vmm::as_select(), + Project::as_select(), + )) + .load_async::<(Sled, Instance, Vmm, Project)>(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(result) + } + pub async fn project_delete_instance( &self, opctx: &OpContext, diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 733e4ef32b..b426c8b472 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -609,6 +609,28 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + /// Retrieve the primary network interface for a given instance. + pub async fn instance_get_primary_network_interface( + &self, + opctx: &OpContext, + authz_instance: &authz::Instance, + ) -> LookupResult { + opctx.authorize(authz::Action::ListChildren, authz_instance).await?; + + use db::schema::instance_network_interface::dsl; + dsl::instance_network_interface + .filter(dsl::time_deleted.is_null()) + .filter(dsl::instance_id.eq(authz_instance.id())) + .filter(dsl::is_primary.eq(true)) + .select(InstanceNetworkInterface::as_select()) + .limit(1) + .first_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + /// Get network interface associated with a given probe. pub async fn probe_get_network_interface( &self, diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index d200f67663..a6a69e388b 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -5,6 +5,7 @@ //! [`DataStore`] methods on [`Vpc`]s. use super::DataStore; +use super::SQL_BATCH_SIZE; use crate::authz; use crate::context::OpContext; use crate::db; @@ -35,6 +36,7 @@ use crate::db::model::VpcSubnetUpdate; use crate::db::model::VpcUpdate; use crate::db::model::{Ipv4Net, Ipv6Net}; use crate::db::pagination::paginated; +use crate::db::pagination::Paginator; use crate::db::queries::vpc::InsertVpcQuery; use crate::db::queries::vpc::VniSearchIter; use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery; @@ -54,6 +56,7 @@ use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::InternalContext; +use omicron_common::api::external::IpNet; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::LookupType; @@ -63,9 +66,13 @@ use omicron_common::api::external::RouteTarget; use omicron_common::api::external::RouterRouteKind as ExternalRouteKind; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::Vni as ExternalVni; +use omicron_common::api::internal::shared::ReifiedVpcRoute; +use omicron_common::api::internal::shared::RouterTarget; use ref_cast::RefCast; use std::collections::BTreeMap; +use std::collections::HashMap; use std::collections::HashSet; +use std::net::IpAddr; use uuid::Uuid; impl DataStore { @@ -1403,6 +1410,279 @@ impl DataStore { }}).await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + + /// Look up a VPC by VNI. + pub async fn vpc_get_system_router( + &self, + opctx: &OpContext, + vpc_id: Uuid, + ) -> LookupResult { + // use db::schema::vpc::dsl as vpc_dsl; + use db::schema::vpc::dsl as vpc_dsl; + use db::schema::vpc_router::dsl as router_dsl; + + vpc_dsl::vpc + .inner_join( + router_dsl::vpc_router + .on(router_dsl::id.eq(vpc_dsl::system_router_id)), + ) + .filter(vpc_dsl::time_deleted.is_null()) + .filter(vpc_dsl::id.eq(vpc_id)) + .filter(router_dsl::time_deleted.is_null()) + .filter(router_dsl::vpc_id.eq(vpc_id)) + .select(VpcRouter::as_select()) + .limit(1) + .first_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Vpc, + LookupType::ById(vpc_id), + ), + ) + }) + } + + /// Fetch all active custom routers (and their parent subnets) + /// in a VPC. + pub async fn vpc_get_active_custom_routers( + &self, + opctx: &OpContext, + vpc_id: Uuid, + ) -> ListResultVec<(VpcSubnet, VpcRouter)> { + use db::schema::vpc_router::dsl as router_dsl; + use db::schema::vpc_subnet::dsl as subnet_dsl; + + subnet_dsl::vpc_subnet + .inner_join( + router_dsl::vpc_router.on(router_dsl::id + .nullable() + .eq(subnet_dsl::custom_router_id)), + ) + .filter(subnet_dsl::time_deleted.is_null()) + .filter(subnet_dsl::vpc_id.is_null()) + .filter(router_dsl::time_deleted.is_null()) + .select((VpcSubnet::as_select(), VpcRouter::as_select())) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Vpc, + LookupType::ById(vpc_id), + ), + ) + }) + } + + /// Resolve all targets in a router into concrete details. + pub async fn vpc_resolve_router_rules( + &self, + opctx: &OpContext, + vpc_router_id: Uuid, + ) -> Result, Error> { + // Get all rules in target router. + opctx.check_complex_operations_allowed()?; + + let (.., authz_project, authz_vpc, authz_router) = + db::lookup::LookupPath::new(opctx, self) + .vpc_router_id(vpc_router_id) + .lookup_for(authz::Action::ListChildren) + .await + .internal_context("lookup built-in services project")?; + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + let mut all_rules = vec![]; + while let Some(p) = paginator.next() { + let batch = self + .vpc_router_route_list( + opctx, + &authz_router, + &PaginatedBy::Id(p.current_pagparams()), + ) + .await?; + paginator = p + .found_batch(&batch, &|s: &nexus_db_model::RouterRoute| s.id()); + all_rules.extend(batch); + } + + // XXX: transaction based on generation number? + let mut subnet_names = HashSet::new(); + let mut vpc_names = HashSet::new(); + let mut inetgw_names = HashSet::new(); + let mut instance_names = HashSet::new(); + for rule in &all_rules { + match &rule.target.0 { + RouteTarget::Vpc(n) => { + vpc_names.insert(n.clone()); + } + RouteTarget::Subnet(n) => { + subnet_names.insert(n.clone()); + } + RouteTarget::Instance(n) => { + instance_names.insert(n.clone()); + } + RouteTarget::InternetGateway(n) => { + inetgw_names.insert(n.clone()); + } + _ => {} + } + + match &rule.destination.0 { + RouteDestination::Vpc(n) => { + vpc_names.insert(n.clone()); + } + RouteDestination::Subnet(n) => { + subnet_names.insert(n.clone()); + } + _ => {} + } + } + + // TODO: transact these, and/or solve in fewer queries. + let mut subnets = HashMap::new(); + for name in subnet_names.drain() { + if let Ok((.., subnet)) = db::lookup::LookupPath::new(opctx, self) + .vpc_id(authz_vpc.id()) + .vpc_subnet_name(Name::ref_cast(&name)) + .fetch() + .await + { + subnets.insert(name, subnet); + } + } + let mut vpcs = HashMap::new(); + for name in vpc_names.drain() { + if let Ok((.., vpc)) = db::lookup::LookupPath::new(opctx, self) + .project_id(authz_project.id()) + .vpc_name(Name::ref_cast(&name)) + .fetch() + .await + { + vpcs.insert(name, vpc); + } + } + let mut instances = HashMap::new(); + for name in instance_names.drain() { + if let Ok((.., authz_instance, instance)) = + db::lookup::LookupPath::new(opctx, self) + .project_id(authz_project.id()) + .instance_name(Name::ref_cast(&name)) + .fetch() + .await + { + // XXX: currently an instance can have one primary, + // and it is not dual-stack (v4 + v6). We need + // to clarify what should be resolved in the v6 case. + if let Ok(primary_nic) = self + .instance_get_primary_network_interface( + opctx, + &authz_instance, + ) + .await + { + instances.insert(name, (instance, primary_nic)); + } + } + } + // TODO: validate names of Internet Gateways. + + // See the discussion in `resolve_firewall_rules_for_sled_agent` on + // how we should resolve name misses in route resolution. + // This method adopts the same strategy: a lookup failure corresponds + // to a NO-OP rule. + let mut out = HashSet::new(); + for rule in all_rules { + // Some dests/targets (e.g., subnet) resolve to *several* specifiers + // to handle both v4 and v6. The user-facing API will prevent severe + // mistakes on naked IPs/CIDRs (mixed v4/6), but we need to be smarter + // around named entities here. + let (v4_dest, v6_dest) = match rule.destination.0 { + RouteDestination::Ip(ip @ IpAddr::V4(_)) => { + (Some(IpNet::single(ip)), None) + } + RouteDestination::Ip(ip @ IpAddr::V6(_)) => { + (None, Some(IpNet::single(ip))) + } + RouteDestination::IpNet(ip @ IpNet::V4(_)) => (Some(ip), None), + RouteDestination::IpNet(ip @ IpNet::V6(_)) => (None, Some(ip)), + RouteDestination::Subnet(n) => subnets + .get(&n) + .map(|s| { + ( + Some(s.ipv4_block.0.into()), + Some(s.ipv6_block.0.into()), + ) + }) + .unwrap_or_default(), + + // TODO: VPC peering. + RouteDestination::Vpc(_) => (None, None), + }; + + let (v4_target, v6_target) = match rule.target.0 { + RouteTarget::Ip(ip @ IpAddr::V4(_)) => { + (Some(RouterTarget::Ip(ip)), None) + } + RouteTarget::Ip(ip @ IpAddr::V6(_)) => { + (None, Some(RouterTarget::Ip(ip))) + } + RouteTarget::Subnet(n) => subnets + .get(&n) + .map(|s| { + ( + Some(RouterTarget::VpcSubnet( + s.ipv4_block.0.into(), + )), + Some(RouterTarget::VpcSubnet( + s.ipv6_block.0.into(), + )), + ) + }) + .unwrap_or_default(), + RouteTarget::Instance(n) => instances + .get(&n) + .map(|i| match i.1.ip { + // TODO: update for dual-stack v4/6. + ip @ IpNetwork::V4(_) => { + (Some(RouterTarget::Ip(ip.ip())), None) + } + ip @ IpNetwork::V6(_) => { + (None, Some(RouterTarget::Ip(ip.ip()))) + } + }) + .unwrap_or_default(), + RouteTarget::Drop => { + (Some(RouterTarget::Drop), Some(RouterTarget::Drop)) + } + + // TODO: Internet Gateways. + // The semantic here is 'name match => allow', + // as the other aspect they will control is SNAT + // IP allocation. Today, presence of this rule + // allows upstream regardless of name. + RouteTarget::InternetGateway(_n) => ( + Some(RouterTarget::InternetGateway), + Some(RouterTarget::InternetGateway), + ), + + // TODO: VPC Peering. + RouteTarget::Vpc(_) => (None, None), + }; + + if let (Some(dest), Some(target)) = (v4_dest, v4_target) { + out.insert(ReifiedVpcRoute { dest, target }); + } + + if let (Some(dest), Some(target)) = (v6_dest, v6_target) { + out.insert(ReifiedVpcRoute { dest, target }); + } + } + + Ok(out) + } } #[cfg(test)] diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index c5c1aaf104..6f21d61204 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -7,19 +7,21 @@ use super::common::BackgroundTask; use futures::future::BoxFuture; use futures::FutureExt; -use nexus_db_model::{Sled, SledState}; +use nexus_db_model::{Sled, SledState, Vni}; use nexus_db_queries::{context::OpContext, db::DataStore}; use nexus_networking::sled_client_from_address; use nexus_types::{ deployment::SledFilter, external_api::views::SledPolicy, identity::Asset, + identity::Resource, }; -use omicron_common::api::{external::Vni, internal::shared::RouterId}; +use omicron_common::api::internal::shared::ReifiedVpcRoute; +use omicron_common::api::internal::shared::{ReifiedVpcRouteSet, RouterId}; use serde_json::json; -use sled_agent_client::types::ReifiedVpcRoute; use std::{ collections::{HashMap, HashSet}, sync::Arc, }; +use uuid::Uuid; pub struct VpcRouteManager { datastore: Arc, @@ -38,6 +40,7 @@ impl BackgroundTask for VpcRouteManager { ) -> BoxFuture<'a, serde_json::Value> { async { let log = &opctx.log; + // let mut paginator = Paginator::new(MAX_SLED_AGENTS); // XX: copied from omicron#5566 let sleds = match self @@ -71,11 +74,12 @@ impl BackgroundTask for VpcRouteManager { .collect(); // XX: actually reify rules. - let mut known_rules: HashMap> = + let mut known_rules: HashMap> = HashMap::new(); + let mut db_routers = HashMap::new(); for (sled, client) in sled_clients { - let Ok(a) = client.list_vpc_routes().await else { + let Ok(route_sets) = client.list_vpc_routes().await else { warn!( log, "failed to fetch current VPC route state from sled"; @@ -84,11 +88,142 @@ impl BackgroundTask for VpcRouteManager { continue; }; - // XX: Who decides what we want? Do we figure out the NICs - // here? Or take the sled at their word for what subnets - // they want? + let route_sets = route_sets.into_inner(); - todo!() + // Lookup all missing (VNI, subnet) pairs we need from this sled. + for set in &route_sets { + let system_route = + RouterId { vni: set.id.vni, subnet: None }; + + if db_routers.contains_key(&system_route) { + continue; + } + + let db_vni = Vni(set.id.vni); + let Ok(vpc) = + self.datastore.resolve_vni_to_vpc(opctx, db_vni).await + else { + error!( + log, + "failed to fetch VPC from VNI"; + "sled" => sled.serial_number(), + "vni" => format!("{db_vni:?}") + ); + continue; + }; + + let vpc_id = vpc.identity().id; + + let Ok(system_router) = self + .datastore + .vpc_get_system_router(opctx, vpc_id) + .await + else { + error!( + log, + "failed to fetch system router for VPC"; + "vpc" => vpc_id.to_string() + ); + continue; + }; + + let Ok(custom_routers) = self + .datastore + .vpc_get_active_custom_routers(opctx, vpc_id) + .await + else { + error!( + log, + "failed to fetch custom routers for VPC"; + "vpc" => vpc_id.to_string() + ); + continue; + }; + + db_routers.insert(system_route, system_router); + db_routers.extend(custom_routers.into_iter().map( + |(subnet, router)| { + ( + RouterId { + vni: set.id.vni, + subnet: Some(subnet.ipv4_block.0.into()), + }, + router, + ) + }, + )); + // XX: do this right / unify v4 and v6 + // db_routers.extend(custom_routers.into_iter().map( + // |(subnet, router)| { + // ( + // RouterId { + // vni: set.id.vni, + // subnet: Some(subnet.ipv6_block.0.into()), + // }, + // router, + // ) + // }, + // )); + } + + let mut to_push = HashMap::new(); + + // reify into known_rules on an as-needed basis. + for set in &route_sets { + let Some(db_router) = db_routers.get(&set.id) else { + // The sled wants to know about rules for a VPC + // subnet with no custom router set. Send them + // the empty list. + to_push.insert(set.id, HashSet::new()); + continue; + }; + + let router_id = db_router.id(); + + // We may have already resolved the rules for this + // router in a previous call. + if let Some(rules) = known_rules.get(&router_id) { + to_push.insert(set.id, rules.clone()); + continue; + } + + match self + .datastore + .vpc_resolve_router_rules( + opctx, + db_router.identity().id, + ) + .await + { + Ok(rules) => { + to_push.insert(set.id, rules.clone()); + known_rules.insert(router_id, rules); + } + Err(e) => { + error!( + &log, + "failed to compute subnet routes"; + "router" => router_id.to_string(), + "err" => e.to_string() + ); + } + } + } + + let to_push = to_push + .into_iter() + .map(|(id, routes)| ReifiedVpcRouteSet { id, routes }) + .collect(); + + if let Err(e) = client.set_vpc_routes(&to_push).await { + error!( + log, + "failed to push new VPC route state from sled"; + "sled" => sled.serial_number(), + "err" => format!("{e}") + ); + continue; + }; } json!({}) diff --git a/schema/crdb/vpc-subnet-routing/up02.sql b/schema/crdb/vpc-subnet-routing/up02.sql new file mode 100644 index 0000000000..e69de29bb2 From 568f44c426284a3489523079fa856684a91119ae Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 17 May 2024 13:47:32 +0100 Subject: [PATCH 10/59] Wrong ID in router initialisation. --- nexus/db-queries/src/db/datastore/vpc.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index a6a69e388b..5cbf0a5edb 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -254,6 +254,7 @@ impl DataStore { opctx: &OpContext, authz_router: &authz::VpcRouter, ) -> Result<(), Error> { + use crate::db::fixed_data::vpc::SERVICES_VPC; use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET; use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET_ROUTE_ID; use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; @@ -293,7 +294,7 @@ impl DataStore { let route = RouterRoute::for_subnet( route_id, - *SERVICES_VPC_ID, + SERVICES_VPC.system_router_id, vpc_subnet.name().clone().into(), ) .expect("builtin service names are short enough for route naming"); From 2a25d74fd88feb42deb05fbbfebb0f7d002ec897 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 17 May 2024 21:10:52 +0100 Subject: [PATCH 11/59] Rule insert/delete with OPTE. --- illumos-utils/src/opte/firewall_rules.rs | 19 +----- illumos-utils/src/opte/mod.rs | 31 +++++++++ illumos-utils/src/opte/port.rs | 9 +++ illumos-utils/src/opte/port_manager.rs | 80 +++++++++++++++++++++--- sled-agent/src/http_entrypoints.rs | 2 +- sled-agent/src/sled_agent.rs | 7 ++- 6 files changed, 119 insertions(+), 29 deletions(-) diff --git a/illumos-utils/src/opte/firewall_rules.rs b/illumos-utils/src/opte/firewall_rules.rs index 02882a226b..6df07b3a54 100644 --- a/illumos-utils/src/opte/firewall_rules.rs +++ b/illumos-utils/src/opte/firewall_rules.rs @@ -4,6 +4,7 @@ //! Convert Omicron VPC firewall rules to OPTE firewall rules. +use super::net_to_cidr; use crate::opte::params::VpcFirewallRule; use crate::opte::Vni; use macaddr::MacAddr6; @@ -19,11 +20,6 @@ use oxide_vpc::api::Filters; use oxide_vpc::api::FirewallAction; use oxide_vpc::api::FirewallRule; use oxide_vpc::api::IpAddr; -use oxide_vpc::api::IpCidr; -use oxide_vpc::api::Ipv4Cidr; -use oxide_vpc::api::Ipv4PrefixLen; -use oxide_vpc::api::Ipv6Cidr; -use oxide_vpc::api::Ipv6PrefixLen; use oxide_vpc::api::Ports; use oxide_vpc::api::ProtoFilter; use oxide_vpc::api::Protocol; @@ -70,23 +66,12 @@ impl FromVpcFirewallRule for VpcFirewallRule { { Address::Ip(IpAddr::Ip4(net.ip().into())) } - HostIdentifier::Ip(IpNet::V4(net)) => { - Address::Subnet(IpCidr::Ip4(Ipv4Cidr::new( - net.ip().into(), - Ipv4PrefixLen::new(net.prefix()).unwrap(), - ))) - } HostIdentifier::Ip(IpNet::V6(net)) if net.prefix() == 128 => { Address::Ip(IpAddr::Ip6(net.ip().into())) } - HostIdentifier::Ip(IpNet::V6(net)) => { - Address::Subnet(IpCidr::Ip6(Ipv6Cidr::new( - net.ip().into(), - Ipv6PrefixLen::new(net.prefix()).unwrap(), - ))) - } + HostIdentifier::Ip(ip) => Address::Subnet(net_to_cidr(*ip)), HostIdentifier::Vpc(vni) => { Address::Vni(Vni::new(u32::from(*vni)).unwrap()) } diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index d06b6b26e5..367a1836c4 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -18,6 +18,14 @@ mod port; mod port_manager; pub use firewall_rules::opte_firewall_rules; +use omicron_common::api::external::IpNet; +use omicron_common::api::internal::shared; +use oxide_vpc::api::IpCidr; +use oxide_vpc::api::Ipv4Cidr; +use oxide_vpc::api::Ipv4PrefixLen; +use oxide_vpc::api::Ipv6Cidr; +use oxide_vpc::api::Ipv6PrefixLen; +use oxide_vpc::api::RouterTarget; pub use port::Port; pub use port_manager::PortManager; pub use port_manager::PortTicket; @@ -63,3 +71,26 @@ impl Gateway { &self.ip } } + +fn net_to_cidr(net: IpNet) -> IpCidr { + match net { + IpNet::V4(net) => IpCidr::Ip4(Ipv4Cidr::new( + net.ip().into(), + Ipv4PrefixLen::new(net.prefix()).unwrap(), + )), + IpNet::V6(net) => IpCidr::Ip6(Ipv6Cidr::new( + net.ip().into(), + Ipv6PrefixLen::new(net.prefix()).unwrap(), + )), + } +} + +fn router_target_opte(target: &shared::RouterTarget) -> RouterTarget { + use shared::RouterTarget::*; + match target { + Drop => RouterTarget::Drop, + InternetGateway => RouterTarget::InternetGateway, + Ip(ip) => RouterTarget::Ip((*ip).into()), + VpcSubnet(net) => RouterTarget::VpcSubnet(net_to_cidr(*net)), + } +} diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 6fbb89c450..013cc5bc61 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -7,6 +7,7 @@ use crate::opte::Gateway; use crate::opte::Vni; use macaddr::MacAddr6; +use omicron_common::api::external::IpNet; use std::net::IpAddr; use std::sync::Arc; @@ -22,6 +23,8 @@ struct PortInner { slot: u8, // Geneve VNI for the VPC vni: Vni, + // Subnet the port belong to within the VPC. + subnet: IpNet, // Information about the virtual gateway, aka OPTE gateway: Gateway, // TODO-remove(#2932): Remove this once we can put Viona directly on top of an @@ -89,6 +92,7 @@ impl Port { mac: MacAddr6, slot: u8, vni: Vni, + subnet: IpNet, gateway: Gateway, vnic: String, ) -> Self { @@ -99,6 +103,7 @@ impl Port { mac, slot, vni, + subnet, gateway, vnic, }), @@ -126,6 +131,10 @@ impl Port { &self.inner.vni } + pub fn subnet(&self) -> &IpNet { + &self.inner.subnet + } + pub fn vnic_name(&self) -> &str { &self.inner.vnic } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 02b85e37bd..dc6e96a5b8 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -22,6 +22,7 @@ use omicron_common::api::internal::shared::RouterId; use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget; use omicron_common::api::internal::shared::SourceNatConfig; use oxide_vpc::api::AddRouterEntryReq; +use oxide_vpc::api::DelRouterEntryReq; use oxide_vpc::api::DhcpCfg; use oxide_vpc::api::ExternalIpCfg; use oxide_vpc::api::IpCfg; @@ -341,6 +342,7 @@ impl PortManager { mac, nic.slot, vni, + nic.subnet, gateway, vnic, ); @@ -452,18 +454,78 @@ impl PortManager { .collect() } - pub fn vpc_routes_ensure(&self, new_routes: Vec) { + pub fn vpc_routes_ensure( + &self, + new_routes: Vec, + ) -> Result<(), Error> { let mut routes = self.inner.routes.lock().unwrap(); - error!( - self.inner.log, - "I got routes I don't know what to do with!"; - "route_set" => format!("{new_routes:?}") - ); - // *routes = new_routes; + let mut deltas = HashMap::new(); + for set in new_routes { + let old = routes.get(&set.id); + + let (to_add, to_delete) = if let Some(old) = old { + ( + set.routes.difference(old).cloned().collect(), + old.difference(&set.routes).cloned().collect(), + ) + } else { + (set.routes.clone(), HashSet::new()) + }; + deltas.insert(set.id, (to_add, to_delete)); + + routes.insert(set.id, set.routes); + } drop(routes); - // XX: compute deltas. - // XX: push down to OPTE. + let ports = self.inner.ports.lock().unwrap(); + #[cfg(target_os = "illumos")] + let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?; + + for port in ports.values() { + let vni = external::Vni::try_from(port.vni().as_u32()).unwrap(); + let system_id = RouterId { vni, subnet: None }; + let system_delta = deltas.get(&system_id); + + let custom_id = RouterId { vni, subnet: Some(*port.subnet()) }; + + let custom_delta = deltas.get(&custom_id); + + #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] + for (class, delta) in [ + (RouterClass::System, system_delta), + (RouterClass::Custom, custom_delta), + ] { + let Some((to_add, to_delete)) = delta else { + continue; + }; + + for route in to_delete { + let route = DelRouterEntryReq { + class, + port_name: port.name().into(), + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + }; + + #[cfg(target_os = "illumos")] + hdl.del_router_entry(&route)?; + } + + for route in to_add { + let route = AddRouterEntryReq { + class, + port_name: port.name().into(), + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + }; + + #[cfg(target_os = "illumos")] + hdl.add_router_entry(&route)?; + } + } + } + + Ok(()) } /// Ensure external IPs for an OPTE port are up to date. diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index f41cd5e817..bf85bb49a7 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -1043,6 +1043,6 @@ async fn set_vpc_routes( body: TypedBody>, ) -> Result { let sa = request_context.context(); - sa.set_vpc_routes(body.into_inner()); + sa.set_vpc_routes(body.into_inner())?; Ok(HttpResponseUpdatedNoContent()) } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 01a6e3e9d7..263db9555e 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -1095,8 +1095,11 @@ impl SledAgent { self.inner.port_manager.vpc_routes_list() } - pub fn set_vpc_routes(&self, routes: Vec) { - self.inner.port_manager.vpc_routes_ensure(routes) + pub fn set_vpc_routes( + &self, + routes: Vec, + ) -> Result<(), Error> { + self.inner.port_manager.vpc_routes_ensure(routes).map_err(Error::from) } /// Return the metric producer registry. From c718c8dc0e6634cec6eea70f6db6b79729ada93d Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 17 May 2024 23:23:21 +0100 Subject: [PATCH 12/59] Correctly instantiate router rules if existing --- illumos-utils/src/opte/port_manager.rs | 138 +++++++++++-------------- sled-agent/src/instance.rs | 1 + sled-agent/src/probe_manager.rs | 1 + sled-agent/src/services.rs | 10 +- 4 files changed, 72 insertions(+), 78 deletions(-) diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index dc6e96a5b8..8e95c4e46b 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -31,7 +31,6 @@ use oxide_vpc::api::Ipv4Cfg; use oxide_vpc::api::Ipv6Cfg; use oxide_vpc::api::MacAddr; use oxide_vpc::api::RouterClass; -use oxide_vpc::api::RouterTarget; use oxide_vpc::api::SNat4Cfg; use oxide_vpc::api::SNat6Cfg; use oxide_vpc::api::SetExternalIpsReq; @@ -120,6 +119,7 @@ impl PortManager { floating_ips: &[IpAddr], firewall_rules: &[VpcFirewallRule], dhcp_config: DhcpCfg, + is_service: bool, ) -> Result<(Port, PortTicket), Error> { let mac = *nic.mac; let vni = Vni::new(nic.vni).unwrap(); @@ -356,88 +356,58 @@ impl PortManager { (port, ticket) }; - // TODO: These should not be filled in like this, and should be informed - // by either our existing knowledge of current knowledge of system + custom - // routers OR we just await the router RPW filling this in for us. - // In future, ∃ VPCs *without* an Internet Gateway so we can't just - // plumb that in as well... - - // Add a router entry for this interface's subnet, directing traffic to the - // VPC subnet. - let route = AddRouterEntryReq { - class: RouterClass::System, - port_name: port_name.clone(), - dest: vpc_subnet, - target: RouterTarget::VpcSubnet(vpc_subnet), - }; - #[cfg(target_os = "illumos")] - hdl.add_router_entry(&route)?; - debug!( - self.inner.log, - "Added VPC Subnet router entry"; - "port_name" => &port_name, - "route" => ?route, - ); - - // TODO-remove - // - // See https://github.com/oxidecomputer/omicron/issues/1336 - // - // This is another part of the workaround, allowing reply traffic from - // the guest back out. Normally, OPTE would drop such traffic at the - // router layer, as it has no route for that external IP address. This - // allows such traffic through. - // - // Note that this exact rule will eventually be included, since it's one - // of the default routing rules in the VPC System Router. However, that - // will likely be communicated in a different way, or could be modified, - // and this specific call should be removed in favor of sending the - // routing rules the control plane provides. - // - // This rule sends all traffic that has no better match to the gateway. - let dest = match vpc_subnet { - IpCidr::Ip4(_) => "0.0.0.0/0", - IpCidr::Ip6(_) => "::/0", - } - .parse() - .unwrap(); - let route = AddRouterEntryReq { - class: RouterClass::System, - port_name: port_name.clone(), - dest, - target: RouterTarget::InternetGateway, - }; - #[cfg(target_os = "illumos")] - hdl.add_router_entry(&route)?; - debug!( - self.inner.log, - "Added default internet gateway route entry"; - "port_name" => &port_name, - "route" => ?route, - ); - - // XX: this is probably not the right initialisation here... - // XX: VPC rules should probably come from ctl plane. - // XX: need to delete safely after. + // XX: need to delete safely after all subnet-holders leave + // to not get flooded with useless rules. let mut routes = self.inner.routes.lock().unwrap(); - routes.entry(RouterId { vni: nic.vni, subnet: None }).or_insert_with( - || { + let system_routes = routes + .entry(RouterId { vni: nic.vni, subnet: None }) + .or_insert_with(|| { let mut out = HashSet::new(); - out.insert(ReifiedVpcRoute { - dest: "0.0.0.0/0".parse().unwrap(), - target: ApiRouterTarget::InternetGateway, - }); - out.insert(ReifiedVpcRoute { - dest: "::/0".parse().unwrap(), - target: ApiRouterTarget::InternetGateway, - }); + // Services do not talk to one another via OPTE, but do need + // to reach out over the Internet *before* nexus is up to give + // us real rules. The easiest bet is to instantiate these here. + if is_service { + out.insert(ReifiedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway, + }); + out.insert(ReifiedVpcRoute { + dest: "::/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway, + }); + } out - }, - ); - routes + }) + .clone(); + + let custom_routes = routes .entry(RouterId { vni: nic.vni, subnet: Some(nic.subnet) }) .or_insert_with(|| HashSet::default()); + for (class, routes) in [ + (RouterClass::System, &system_routes), + (RouterClass::Custom, custom_routes), + ] { + for route in routes { + let route = AddRouterEntryReq { + class, + port_name: port_name.clone(), + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + }; + + #[cfg(target_os = "illumos")] + hdl.add_router_entry(&route)?; + + debug!( + self.inner.log, + "Added router entry"; + "port_name" => &port_name, + "route" => ?route, + ); + } + } + info!( self.inner.log, "Created OPTE port"; @@ -509,6 +479,13 @@ impl PortManager { #[cfg(target_os = "illumos")] hdl.del_router_entry(&route)?; + + debug!( + self.inner.log, + "Removed router entry"; + "port_name" => &port.name(), + "route" => ?route, + ); } for route in to_add { @@ -521,6 +498,13 @@ impl PortManager { #[cfg(target_os = "illumos")] hdl.add_router_entry(&route)?; + + debug!( + self.inner.log, + "Added router entry"; + "port_name" => &port.name(), + "route" => ?route, + ); } } } diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 271eceb556..e43e13f96f 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -1330,6 +1330,7 @@ impl InstanceRunner { floating_ips, &self.firewall_rules, self.dhcp_config.clone(), + false, )?; opte_ports.push(port); } diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index 16559039a2..67357c9512 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -239,6 +239,7 @@ impl ProbeManagerInner { priority: VpcFirewallRulePriority(100), }], DhcpCfg::default(), + false, )?; let installed_zone = ZoneBuilderFactory::default() diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index c9a5014402..b8bde2bfc8 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1266,7 +1266,15 @@ impl ServiceManager { // config allows outbound access which is enough for // Boundary NTP which needs to come up before Nexus. let port = port_manager - .create_port(nic, snat, None, floating_ips, &[], DhcpCfg::default()) + .create_port( + nic, + snat, + None, + floating_ips, + &[], + DhcpCfg::default(), + true, + ) .map_err(|err| Error::ServicePortCreation { service: zone_type_str.clone(), err: Box::new(err), From 06eaaabb60f25818ba78cdad5981dcab1b0c2a02 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Mon, 20 May 2024 11:58:13 +0100 Subject: [PATCH 13/59] Feed Clippy. --- illumos-utils/src/opte/mod.rs | 1 + illumos-utils/src/opte/port.rs | 68 ++++++++++++-------------- illumos-utils/src/opte/port_manager.rs | 43 +++++++++++----- sled-agent/src/instance.rs | 14 +++--- sled-agent/src/probe_manager.rs | 20 ++++---- sled-agent/src/services.rs | 16 +++--- 6 files changed, 86 insertions(+), 76 deletions(-) diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 367a1836c4..f6ef186808 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -27,6 +27,7 @@ use oxide_vpc::api::Ipv6Cidr; use oxide_vpc::api::Ipv6PrefixLen; use oxide_vpc::api::RouterTarget; pub use port::Port; +pub use port_manager::PortCreateParams; pub use port_manager::PortManager; pub use port_manager::PortTicket; diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 013cc5bc61..38ba1b8c1c 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -12,21 +12,22 @@ use std::net::IpAddr; use std::sync::Arc; #[derive(Debug)] -struct PortInner { - // Name of the port as identified by OPTE - name: String, - // IP address within the VPC Subnet - ip: IpAddr, - // VPC-private MAC address - mac: MacAddr6, - // Emulated PCI slot for the guest NIC, passed to Propolis - slot: u8, - // Geneve VNI for the VPC - vni: Vni, - // Subnet the port belong to within the VPC. - subnet: IpNet, - // Information about the virtual gateway, aka OPTE - gateway: Gateway, +pub struct PortData { + /// Name of the port as identified by OPTE + pub(crate) name: String, + /// IP address within the VPC Subnet + pub(crate) ip: IpAddr, + /// VPC-private MAC address + pub(crate) mac: MacAddr6, + /// Emulated PCI slot for the guest NIC, passed to Propolis + pub(crate) slot: u8, + /// Geneve VNI for the VPC + pub(crate) vni: Vni, + /// Subnet the port belong to within the VPC. + pub(crate) subnet: IpNet, + /// Information about the virtual gateway, aka OPTE + pub(crate) gateway: Gateway, + /// Name of the VNIC the OPTE port is bound to. // TODO-remove(#2932): Remove this once we can put Viona directly on top of an // OPTE port device. // @@ -36,7 +37,18 @@ struct PortInner { // https://github.com/oxidecomputer/opte/issues/178 for more details. This // can be changed back to a real VNIC when that is resolved, and the Drop // impl below can simplify to just call `drop(self.vnic)`. - vnic: String, + pub(crate) vnic: String, +} + +#[derive(Debug)] +struct PortInner(PortData); + +impl core::ops::Deref for PortInner { + type Target = PortData; + + fn deref(&self) -> &Self::Target { + &self.0 + } } #[cfg(target_os = "illumos")] @@ -86,28 +98,8 @@ pub struct Port { } impl Port { - pub fn new( - name: String, - ip: IpAddr, - mac: MacAddr6, - slot: u8, - vni: Vni, - subnet: IpNet, - gateway: Gateway, - vnic: String, - ) -> Self { - Self { - inner: Arc::new(PortInner { - name, - ip, - mac, - slot, - vni, - subnet, - gateway, - vnic, - }), - } + pub fn new(data: PortData) -> Self { + Self { inner: Arc::new(PortInner(data)) } } pub fn ip(&self) -> &IpAddr { diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 8e95c4e46b..968854381e 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -8,6 +8,7 @@ use crate::opte::opte_firewall_rules; use crate::opte::params::DeleteVirtualNetworkInterfaceHost; use crate::opte::params::SetVirtualNetworkInterfaceHost; use crate::opte::params::VpcFirewallRule; +use crate::opte::port::PortData; use crate::opte::Error; use crate::opte::Gateway; use crate::opte::Port; @@ -85,6 +86,18 @@ impl PortManagerInner { } } +#[derive(Debug)] +/// Parameters needed to create and configure an OPTE port. +pub struct PortCreateParams<'a> { + pub nic: &'a NetworkInterface, + pub source_nat: Option, + pub ephemeral_ip: Option, + pub floating_ips: &'a [IpAddr], + pub firewall_rules: &'a [VpcFirewallRule], + pub dhcp_config: DhcpCfg, + pub is_service: bool, +} + /// The port manager controls all OPTE ports on a single host. #[derive(Debug, Clone)] pub struct PortManager { @@ -113,14 +126,18 @@ impl PortManager { #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] pub fn create_port( &self, - nic: &NetworkInterface, - source_nat: Option, - ephemeral_ip: Option, - floating_ips: &[IpAddr], - firewall_rules: &[VpcFirewallRule], - dhcp_config: DhcpCfg, - is_service: bool, + params: PortCreateParams, ) -> Result<(Port, PortTicket), Error> { + let PortCreateParams { + nic, + source_nat, + ephemeral_ip, + floating_ips, + firewall_rules, + dhcp_config, + is_service, + } = params; + let mac = *nic.mac; let vni = Vni::new(nic.vni).unwrap(); let subnet = IpNetwork::from(nic.subnet); @@ -336,16 +353,16 @@ impl PortManager { let (port, ticket) = { let mut ports = self.inner.ports.lock().unwrap(); let ticket = PortTicket::new(nic.id, nic.kind, self.inner.clone()); - let port = Port::new( - port_name.clone(), - nic.ip, + let port = Port::new(PortData { + name: port_name.clone(), + ip: nic.ip, mac, - nic.slot, + slot: nic.slot, vni, - nic.subnet, + subnet: nic.subnet, gateway, vnic, - ); + }); let old = ports.insert((nic.id, nic.kind), port.clone()); assert!( old.is_none(), diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index e43e13f96f..df5970e634 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -27,7 +27,7 @@ use backoff::BackoffError; use chrono::Utc; use illumos_utils::dladm::Etherstub; use illumos_utils::link::VnicAllocator; -use illumos_utils::opte::{DhcpCfg, PortManager}; +use illumos_utils::opte::{DhcpCfg, PortCreateParams, PortManager}; use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory}; use illumos_utils::svc::wait_for_service; use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; @@ -1323,15 +1323,15 @@ impl InstanceRunner { } else { (None, None, &[][..]) }; - let port = self.port_manager.create_port( + let port = self.port_manager.create_port(PortCreateParams { nic, - snat, + source_nat: snat, ephemeral_ip, floating_ips, - &self.firewall_rules, - self.dhcp_config.clone(), - false, - )?; + firewall_rules: &self.firewall_rules, + dhcp_config: self.dhcp_config.clone(), + is_service: false, + })?; opte_ports.push(port); } diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index 67357c9512..cc38725b61 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -3,7 +3,7 @@ use anyhow::{anyhow, Result}; use illumos_utils::dladm::Etherstub; use illumos_utils::link::VnicAllocator; use illumos_utils::opte::params::VpcFirewallRule; -use illumos_utils::opte::{DhcpCfg, PortManager}; +use illumos_utils::opte::{DhcpCfg, PortCreateParams, PortManager}; use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory}; use illumos_utils::zone::Zones; use nexus_client::types::{ProbeExternalIp, ProbeInfo}; @@ -223,12 +223,12 @@ impl ProbeManagerInner { .get(0) .ok_or(anyhow!("expected an external ip"))?; - let port = self.port_manager.create_port( - &nic, - None, - Some(eip.ip), - &[], // floating ips - &[VpcFirewallRule { + let port = self.port_manager.create_port(PortCreateParams { + nic, + source_nat: None, + ephemeral_ip: Some(eip.ip), + floating_ips: &[], + firewall_rules: &[VpcFirewallRule { status: VpcFirewallRuleStatus::Enabled, direction: VpcFirewallRuleDirection::Inbound, targets: vec![nic.clone()], @@ -238,9 +238,9 @@ impl ProbeManagerInner { action: VpcFirewallRuleAction::Allow, priority: VpcFirewallRulePriority(100), }], - DhcpCfg::default(), - false, - )?; + dhcp_config: DhcpCfg::default(), + is_service: false, + })?; let installed_zone = ZoneBuilderFactory::default() .builder() diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index b8bde2bfc8..325eb5cd61 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -49,7 +49,7 @@ use illumos_utils::dladm::{ Dladm, Etherstub, EtherstubVnic, GetSimnetError, PhysicalLink, }; use illumos_utils::link::{Link, VnicAllocator}; -use illumos_utils::opte::{DhcpCfg, Port, PortManager, PortTicket}; +use illumos_utils::opte::{DhcpCfg, Port, PortCreateParams, PortManager, PortTicket}; use illumos_utils::running_zone::{ EnsureAddressError, InstalledZone, RunCommandError, RunningZone, ZoneBuilderFactory, @@ -1266,15 +1266,15 @@ impl ServiceManager { // config allows outbound access which is enough for // Boundary NTP which needs to come up before Nexus. let port = port_manager - .create_port( + .create_port(PortCreateParams { nic, - snat, - None, + source_nat: snat, + ephemeral_ip: None, floating_ips, - &[], - DhcpCfg::default(), - true, - ) + firewall_rules: &[], + dhcp_config: DhcpCfg::default(), + is_service: true, + }) .map_err(|err| Error::ServicePortCreation { service: zone_type_str.clone(), err: Box::new(err), From 38beadd65dd3b832e4e5d382619f87a6321a7b45 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Mon, 20 May 2024 12:00:23 +0100 Subject: [PATCH 14/59] Comment adapted. --- sled-agent/src/services.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 325eb5cd61..ceaf8bec05 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1262,7 +1262,7 @@ impl ServiceManager { // Create the OPTE port for the service. // Note we don't plumb any firewall rules at this point, - // Nexus will plumb them down later but the default OPTE + // Nexus will plumb them down later but services' default OPTE // config allows outbound access which is enough for // Boundary NTP which needs to come up before Nexus. let port = port_manager From 17489cbef51bbb9b2b1113153f2fe6dcab7ee20c Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Mon, 20 May 2024 17:33:26 +0100 Subject: [PATCH 15/59] The backing code for a generational RPW Currently there are no triggers attached to most of the operations that will cause us to either a) push or b) re-resolve VPC routes, but this lays the basis for sled-agent and the background task to talk in terms of versions. --- common/src/api/internal/shared.rs | 28 ++- illumos-utils/src/opte/port_manager.rs | 62 ++++-- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/vpc_router.rs | 9 +- nexus/db-queries/src/db/datastore/vpc.rs | 251 +++++++++++++++++++++++ nexus/src/app/background/vpc_routes.rs | 128 +++++++----- openapi/sled-agent.json | 55 ++++- schema/crdb/dbinit.sql | 8 +- schema/crdb/vpc-subnet-routing/up02.sql | 7 + sled-agent/src/http_entrypoints.rs | 8 +- sled-agent/src/services.rs | 4 +- sled-agent/src/sled_agent.rs | 4 +- 12 files changed, 481 insertions(+), 84 deletions(-) diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 96f0fecd95..52601a01a8 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -611,7 +611,23 @@ pub enum RouterTarget { VpcSubnet(IpNet), } -/// XX +/// XXX +#[derive( + Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +pub struct RouterVersion { + pub router_id: Uuid, + pub generation: u64, +} + +impl RouterVersion { + pub fn is_replaced_by(&self, other: &Self) -> bool { + (self.router_id != other.router_id) + || self.generation < other.generation + } +} + +/// Implementation details on XXX #[derive( Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] @@ -620,10 +636,18 @@ pub struct RouterId { pub subnet: Option, } -/// XX +/// Version information +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +pub struct ReifiedVpcRouteState { + pub id: RouterId, + pub version: Option, +} + +/// An updated set of routes for a given #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] pub struct ReifiedVpcRouteSet { pub id: RouterId, + pub version: Option, pub routes: HashSet, } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 968854381e..d206273c48 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -19,8 +19,10 @@ use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_common::api::internal::shared::ReifiedVpcRoute; use omicron_common::api::internal::shared::ReifiedVpcRouteSet; +use omicron_common::api::internal::shared::ReifiedVpcRouteState; use omicron_common::api::internal::shared::RouterId; use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget; +use omicron_common::api::internal::shared::RouterVersion; use omicron_common::api::internal::shared::SourceNatConfig; use oxide_vpc::api::AddRouterEntryReq; use oxide_vpc::api::DelRouterEntryReq; @@ -54,6 +56,12 @@ use uuid::Uuid; // Prefix used to identify xde data links. const XDE_LINK_PREFIX: &str = "opte"; +#[derive(Debug, Clone)] +struct RouteSet { + version: Option, + routes: HashSet, +} + #[derive(Debug)] struct PortManagerInner { log: Logger, @@ -68,12 +76,11 @@ struct PortManagerInner { // (which includes the Uuid of the parent instance or service) ports: Mutex>, - // XX: vs. Hashmap? // XX: Should this be the UUID of the VPC? The rulesets are // arguably shared v4+v6, although today we don't yet // allow dual-stack, let alone v6. - // Map of all current resolved routes - routes: Mutex>>, + // Map of all current resolved routes. + routes: Mutex>, } impl PortManagerInner { @@ -379,33 +386,38 @@ impl PortManager { let system_routes = routes .entry(RouterId { vni: nic.vni, subnet: None }) .or_insert_with(|| { - let mut out = HashSet::new(); + let mut routes = HashSet::new(); + // Services do not talk to one another via OPTE, but do need // to reach out over the Internet *before* nexus is up to give // us real rules. The easiest bet is to instantiate these here. if is_service { - out.insert(ReifiedVpcRoute { + routes.insert(ReifiedVpcRoute { dest: "0.0.0.0/0".parse().unwrap(), target: ApiRouterTarget::InternetGateway, }); - out.insert(ReifiedVpcRoute { + routes.insert(ReifiedVpcRoute { dest: "::/0".parse().unwrap(), target: ApiRouterTarget::InternetGateway, }); } - out + + RouteSet { version: None, routes } }) .clone(); let custom_routes = routes .entry(RouterId { vni: nic.vni, subnet: Some(nic.subnet) }) - .or_insert_with(|| HashSet::default()); + .or_insert_with(|| RouteSet { + version: None, + routes: HashSet::default(), + }); for (class, routes) in [ (RouterClass::System, &system_routes), (RouterClass::Custom, custom_routes), ] { - for route in routes { + for route in &routes.routes { let route = AddRouterEntryReq { class, port_name: port_name.clone(), @@ -433,11 +445,11 @@ impl PortManager { Ok((port, ticket)) } - pub fn vpc_routes_list(&self) -> Vec { + pub fn vpc_routes_list(&self) -> Vec { let routes = self.inner.routes.lock().unwrap(); routes .iter() - .map(|(k, v)| ReifiedVpcRouteSet { id: *k, routes: v.clone() }) + .map(|(k, v)| ReifiedVpcRouteState { id: *k, version: v.version }) .collect() } @@ -448,19 +460,31 @@ impl PortManager { let mut routes = self.inner.routes.lock().unwrap(); let mut deltas = HashMap::new(); for set in new_routes { - let old = routes.get(&set.id); - - let (to_add, to_delete) = if let Some(old) = old { - ( - set.routes.difference(old).cloned().collect(), - old.difference(&set.routes).cloned().collect(), - ) + // We have to handle subnet router changes, as well as + // spurious updates from multiple Nexus instances. + // If there's a UUID match, only update if vers increased, + // otherwise take the update verbatim (including loss of version). + let (to_add, to_delete) = if let Some(old) = routes.get(&set.id) { + match (old.version, set.version) { + (Some(old_vers), Some(new_vers)) + if !old_vers.is_replaced_by(&new_vers) => + { + continue; + } + _ => ( + set.routes.difference(&old.routes).cloned().collect(), + old.routes.difference(&set.routes).cloned().collect(), + ), + } } else { (set.routes.clone(), HashSet::new()) }; deltas.insert(set.id, (to_add, to_delete)); - routes.insert(set.id, set.routes); + routes.insert( + set.id, + RouteSet { version: set.version, routes: set.routes }, + ); } drop(routes); diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 95d372167e..bab18d9fae 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1114,6 +1114,7 @@ table! { kind -> crate::VpcRouterKindEnum, vpc_id -> Uuid, rcgen -> Int8, + resolved_version -> Int8, } } diff --git a/nexus/db-model/src/vpc_router.rs b/nexus/db-model/src/vpc_router.rs index 71c753e6aa..51409c38d5 100644 --- a/nexus/db-model/src/vpc_router.rs +++ b/nexus/db-model/src/vpc_router.rs @@ -44,6 +44,7 @@ pub struct VpcRouter { pub vpc_id: Uuid, pub kind: VpcRouterKind, pub rcgen: Generation, + pub resolved_version: i64, } impl VpcRouter { @@ -54,7 +55,13 @@ impl VpcRouter { params: params::VpcRouterCreate, ) -> Self { let identity = VpcRouterIdentity::new(router_id, params.identity); - Self { identity, vpc_id, kind, rcgen: Generation::new() } + Self { + identity, + vpc_id, + kind, + rcgen: Generation::new(), + resolved_version: 0, + } } } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 5cbf0a5edb..69972a701c 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1696,6 +1696,7 @@ mod tests { use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; use crate::db::model::Project; use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; + use ipnetwork::Ipv4Network; use nexus_db_model::IncompleteNetworkInterface; use nexus_db_model::SledUpdate; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; @@ -2211,4 +2212,254 @@ mod tests { db.cleanup().await.unwrap(); logctx.cleanup_successful(); } + + // Test to verify that subnet CRUD operations are correctly + // reflected in the nexus-managed system router attached to a VPC. + #[tokio::test] + async fn test_vpc_system_router_sync_to_subnets() { + usdt::register_probes().unwrap(); + let logctx = + dev::test_setup_log("test_vpc_system_router_sync_to_subnets"); + let log = &logctx.log; + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // Create a project and VPC. + let project_params = params::ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "project".parse().unwrap(), + description: String::from("test project"), + }, + }; + let project = Project::new(Uuid::new_v4(), project_params); + let (authz_project, _) = datastore + .project_create(&opctx, project) + .await + .expect("failed to create project"); + + let vpc_name: external::Name = "my-vpc".parse().unwrap(); + let description = String::from("test vpc"); + let mut incomplete_vpc = IncompleteVpc::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::VpcCreate { + identity: IdentityMetadataCreateParams { + name: vpc_name.clone(), + description: description.clone(), + }, + ipv6_prefix: None, + dns_name: vpc_name.clone(), + }, + ) + .expect("failed to create incomplete VPC"); + let this_vni = Vni(external::Vni::try_from(2048).unwrap()); + incomplete_vpc.vni = this_vni; + info!( + log, + "creating initial VPC"; + "vni" => ?this_vni, + ); + let query = InsertVpcQuery::new(incomplete_vpc); + let (authz_vpc, db_vpc) = datastore + .project_create_vpc_raw(&opctx, &authz_project, query) + .await + .expect("failed to create initial set of VPCs") + .expect("expected an actual VPC"); + info!( + log, + "created VPC"; + "vpc" => ?db_vpc, + ); + + // Now create the system router for this VPC. Subnet CRUD + // operations need this defined to succeed. + let router = VpcRouter::new( + db_vpc.system_router_id, + db_vpc.id(), + VpcRouterKind::System, + nexus_types::external_api::params::VpcRouterCreate { + identity: IdentityMetadataCreateParams { + name: "system".parse().unwrap(), + description: description.clone(), + }, + }, + ); + + let (_, db_router) = datastore + .vpc_create_router(&opctx, &authz_vpc, router) + .await + .unwrap(); + + // InternetGateway route creation is handled by the saga proper, + // so we'll only have subnet routes here. Initially, we start with none: + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[], + ) + .await; + + // Add a new subnet and we should get a new route. + let ipv6_block = db_vpc + .ipv6_prefix + .random_subnet(external::Ipv6Net::VPC_SUBNET_IPV6_PREFIX_LENGTH) + .map(|block| block.0) + .unwrap(); + let (authz_sub0, sub0) = datastore + .vpc_create_subnet( + &opctx, + &authz_vpc, + db::model::VpcSubnet::new( + Uuid::new_v4(), + db_vpc.id(), + IdentityMetadataCreateParams { + name: "s0".parse().unwrap(), + description: "The default subnet...".into(), + }, + external::Ipv4Net( + Ipv4Network::new( + core::net::Ipv4Addr::new(172, 30, 0, 0), + 22, + ) + .unwrap(), + ), + ipv6_block, + ), + ) + .await + .unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub0], + ) + .await; + + // Add another, and get another route. + let ipv6_block = db_vpc + .ipv6_prefix + .random_subnet(external::Ipv6Net::VPC_SUBNET_IPV6_PREFIX_LENGTH) + .map(|block| block.0) + .unwrap(); + let (_, sub1) = datastore + .vpc_create_subnet( + &opctx, + &authz_vpc, + db::model::VpcSubnet::new( + Uuid::new_v4(), + db_vpc.id(), + IdentityMetadataCreateParams { + name: "s1".parse().unwrap(), + description: "A second subnet...".into(), + }, + external::Ipv4Net( + Ipv4Network::new( + core::net::Ipv4Addr::new(172, 31, 0, 0), + 22, + ) + .unwrap(), + ), + ipv6_block, + ), + ) + .await + .unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub0, &sub1], + ) + .await; + + // Rename one subnet, and our invariants should hold. + let sub0 = datastore + .vpc_update_subnet( + &opctx, + &authz_sub0, + VpcSubnetUpdate { + name: Some( + "a-new-name".parse::().unwrap().into(), + ), + description: None, + time_modified: Utc::now(), + }, + ) + .await + .unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub0, &sub1], + ) + .await; + + // Delete one, and routes should stay in sync. + datastore.vpc_delete_subnet(&opctx, &sub0, &authz_sub0).await.unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub1], + ) + .await; + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + + async fn verify_all_subnet_routes_in_router( + opctx: &OpContext, + datastore: &DataStore, + router_id: Uuid, + subnets: &[&VpcSubnet], + ) -> Vec { + let conn = datastore.pool_connection_authorized(opctx).await.unwrap(); + + use db::schema::router_route::dsl; + let routes = dsl::router_route + .filter(dsl::time_deleted.is_null()) + .filter(dsl::vpc_router_id.eq(router_id)) + .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .select(RouterRoute::as_select()) + .load_async(&*conn) + .await + .unwrap(); + + // We should have exactly as many subnet routes as subnets. + assert_eq!(routes.len(), subnets.len()); + + let mut names: HashMap<_, _> = + subnets.iter().map(|s| (s.name().clone(), 0usize)).collect(); + + // Each should have a target+dest bound to a subnet by name. + for route in &routes { + let found_name = match &route.target.0 { + RouteTarget::Subnet(name) => name, + e => panic!("found target {e:?} instead of Subnet({{name}})"), + }; + + match &route.destination.0 { + RouteDestination::Subnet(name) => assert_eq!(name, found_name), + e => panic!("found dest {e:?} instead of Subnet({{name}})"), + } + + *names.get_mut(found_name).unwrap() += 1; + } + + // Each name should be used exactly once. + for (name, count) in names { + assert_eq!(count, 1, "subnet {name} should appear exactly once") + } + + routes + } } diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index 6f21d61204..d4d6478c93 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -14,9 +14,11 @@ use nexus_types::{ deployment::SledFilter, external_api::views::SledPolicy, identity::Asset, identity::Resource, }; -use omicron_common::api::internal::shared::ReifiedVpcRoute; -use omicron_common::api::internal::shared::{ReifiedVpcRouteSet, RouterId}; +use omicron_common::api::internal::shared::{ + ReifiedVpcRoute, ReifiedVpcRouteSet, RouterId, RouterVersion, +}; use serde_json::json; +use std::collections::hash_map::Entry; use std::{ collections::{HashMap, HashSet}, sync::Arc, @@ -33,6 +35,10 @@ impl VpcRouteManager { } } +// There's a sort of eventual consistency happening here. +// ... DETAIL XX ... +// version bumps must happen AFTER other changes occur in +// children etc. to keep this sane and working. :) impl BackgroundTask for VpcRouteManager { fn activate<'a>( &'a mut self, @@ -40,7 +46,6 @@ impl BackgroundTask for VpcRouteManager { ) -> BoxFuture<'a, serde_json::Value> { async { let log = &opctx.log; - // let mut paginator = Paginator::new(MAX_SLED_AGENTS); // XX: copied from omicron#5566 let sleds = match self @@ -73,10 +78,10 @@ impl BackgroundTask for VpcRouteManager { }) .collect(); - // XX: actually reify rules. let mut known_rules: HashMap> = HashMap::new(); let mut db_routers = HashMap::new(); + let mut vni_to_vpc = HashMap::new(); for (sled, client) in sled_clients { let Ok(route_sets) = client.list_vpc_routes().await else { @@ -90,26 +95,35 @@ impl BackgroundTask for VpcRouteManager { let route_sets = route_sets.into_inner(); - // Lookup all missing (VNI, subnet) pairs we need from this sled. + // Lookup all VPC<->Subnet<->Router associations we might need, + // based on the set of VNIs reported by this sled. + // These provide the versions we'll stick with -- in the worst + // case we push newer state to a sled with an older generation + // number, which for set in &route_sets { - let system_route = - RouterId { vni: set.id.vni, subnet: None }; - - if db_routers.contains_key(&system_route) { - continue; - } - let db_vni = Vni(set.id.vni); - let Ok(vpc) = - self.datastore.resolve_vni_to_vpc(opctx, db_vni).await - else { - error!( - log, - "failed to fetch VPC from VNI"; - "sled" => sled.serial_number(), - "vni" => format!("{db_vni:?}") - ); - continue; + let maybe_vpc = vni_to_vpc.entry(set.id.vni); + let vpc = match maybe_vpc { + Entry::Occupied(_) => { + continue; + } + Entry::Vacant(v) => { + let Ok(vpc) = self + .datastore + .resolve_vni_to_vpc(opctx, db_vni) + .await + else { + error!( + log, + "failed to fetch VPC from VNI"; + "sled" => sled.serial_number(), + "vni" => ?db_vni + ); + continue; + }; + + v.insert(vpc) + } }; let vpc_id = vpc.identity().id; @@ -140,50 +154,71 @@ impl BackgroundTask for VpcRouteManager { continue; }; - db_routers.insert(system_route, system_router); - db_routers.extend(custom_routers.into_iter().map( + db_routers.insert( + RouterId { vni: set.id.vni, subnet: None }, + system_router, + ); + db_routers.extend(custom_routers.iter().map( |(subnet, router)| { ( RouterId { vni: set.id.vni, subnet: Some(subnet.ipv4_block.0.into()), }, + router.clone(), + ) + }, + )); + db_routers.extend(custom_routers.into_iter().map( + |(subnet, router)| { + ( + RouterId { + vni: set.id.vni, + subnet: Some(subnet.ipv6_block.0.into()), + }, router, ) }, )); - // XX: do this right / unify v4 and v6 - // db_routers.extend(custom_routers.into_iter().map( - // |(subnet, router)| { - // ( - // RouterId { - // vni: set.id.vni, - // subnet: Some(subnet.ipv6_block.0.into()), - // }, - // router, - // ) - // }, - // )); } - let mut to_push = HashMap::new(); + let mut to_push = Vec::new(); + let mut set_rules = |id, version, routes| { + to_push.push(ReifiedVpcRouteSet { id, routes, version }); + }; - // reify into known_rules on an as-needed basis. + // resolve into known_rules on an as-needed basis. for set in &route_sets { let Some(db_router) = db_routers.get(&set.id) else { // The sled wants to know about rules for a VPC // subnet with no custom router set. Send them - // the empty list. - to_push.insert(set.id, HashSet::new()); + // the empty list, unset its table version. + set_rules(set.id, None, HashSet::new()); continue; }; let router_id = db_router.id(); + let version = RouterVersion { + generation: db_router.resolved_version as u64, + router_id, + }; + + // Only attempt to resolve/push a ruleset if we have a different + // router ID than the sled, or a higher version number. + match &set.version { + Some(v) + if v.router_id == router_id + && v.generation >= version.generation => + { + continue; + } + _ => {} + } // We may have already resolved the rules for this - // router in a previous call. + // router in a previous iteration. if let Some(rules) = known_rules.get(&router_id) { - to_push.insert(set.id, rules.clone()); + set_rules(set.id, Some(version), rules.clone()); continue; } @@ -196,7 +231,7 @@ impl BackgroundTask for VpcRouteManager { .await { Ok(rules) => { - to_push.insert(set.id, rules.clone()); + set_rules(set.id, Some(version), rules.clone()); known_rules.insert(router_id, rules); } Err(e) => { @@ -210,17 +245,12 @@ impl BackgroundTask for VpcRouteManager { } } - let to_push = to_push - .into_iter() - .map(|(id, routes)| ReifiedVpcRouteSet { id, routes }) - .collect(); - if let Err(e) = client.set_vpc_routes(&to_push).await { error!( log, "failed to push new VPC route state from sled"; "sled" => sled.serial_number(), - "err" => format!("{e}") + "err" => ?e ); continue; }; diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index cf55e8ef74..e4778bf06a 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -955,7 +955,7 @@ }, "/vpc-routes": { "get": { - "summary": "Get the current state of VPC routing rules.", + "summary": "Get the current versions of VPC routing rules.", "operationId": "list_vpc_routes", "responses": { "200": { @@ -963,10 +963,10 @@ "content": { "application/json": { "schema": { - "title": "Array_of_ReifiedVpcRouteSet", + "title": "Array_of_ReifiedVpcRouteState", "type": "array", "items": { - "$ref": "#/components/schemas/ReifiedVpcRouteSet" + "$ref": "#/components/schemas/ReifiedVpcRouteState" } } } @@ -4326,7 +4326,7 @@ ] }, "ReifiedVpcRouteSet": { - "description": "XX", + "description": "An updated set of routes for a given", "type": "object", "properties": { "id": { @@ -4338,11 +4338,35 @@ "$ref": "#/components/schemas/ReifiedVpcRoute" }, "uniqueItems": true + }, + "version": { + "$ref": "#/components/schemas/RouterVersion" } }, "required": [ "id", - "routes" + "routes", + "version" + ] + }, + "ReifiedVpcRouteState": { + "description": "Version information", + "type": "object", + "properties": { + "id": { + "$ref": "#/components/schemas/RouterId" + }, + "version": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RouterVersion" + } + ] + } + }, + "required": [ + "id" ] }, "RouteConfig": { @@ -4375,7 +4399,7 @@ ] }, "RouterId": { - "description": "XX", + "description": "Implementation details on XXX", "type": "object", "properties": { "subnet": { @@ -4464,6 +4488,25 @@ } ] }, + "RouterVersion": { + "description": "XXX", + "type": "object", + "properties": { + "generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "router_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "generation", + "router_id" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 486ae68240..9466c04d15 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -1599,7 +1599,13 @@ CREATE TABLE IF NOT EXISTS omicron.public.vpc_router ( time_deleted TIMESTAMPTZ, kind omicron.public.vpc_router_kind NOT NULL, vpc_id UUID NOT NULL, - rcgen INT NOT NULL + rcgen INT NOT NULL, + /* + * version information used to trigger VPC router RPW. + * this is sensitive to CRUD on named resources beyond + * routers e.g. instances, subnets, ... + */ + resolved_version INT NOT NULL DEFAULT 0 ); CREATE UNIQUE INDEX IF NOT EXISTS lookup_router_by_vpc ON omicron.public.vpc_router ( diff --git a/schema/crdb/vpc-subnet-routing/up02.sql b/schema/crdb/vpc-subnet-routing/up02.sql index e69de29bb2..77e72961a3 100644 --- a/schema/crdb/vpc-subnet-routing/up02.sql +++ b/schema/crdb/vpc-subnet-routing/up02.sql @@ -0,0 +1,7 @@ +/* + * version information used to trigger VPC router RPW. + * this is sensitive to CRUD on named resources beyond + * routers e.g. instances, subnets, ... + */ +ALTER TABLE omicron.public.vpc_router +ADD COLUMN IF NOT EXISTS resolved_version INT NOT NULL DEFAULT 0; diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index bf85bb49a7..f8156a617e 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -33,7 +33,9 @@ use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::{ DiskRuntimeState, SledInstanceState, UpdateArtifactId, }; -use omicron_common::api::internal::shared::{ReifiedVpcRouteSet, SwitchPorts}; +use omicron_common::api::internal::shared::{ + ReifiedVpcRouteSet, ReifiedVpcRouteState, SwitchPorts, +}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_hardware::DiskVariant; @@ -1021,14 +1023,14 @@ async fn bootstore_status( Ok(HttpResponseOk(status)) } -/// Get the current state of VPC routing rules. +/// Get the current versions of VPC routing rules. #[endpoint { method = GET, path = "/vpc-routes", }] async fn list_vpc_routes( request_context: RequestContext, -) -> Result>, HttpError> { +) -> Result>, HttpError> { let sa = request_context.context(); Ok(HttpResponseOk(sa.list_vpc_routes())) } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index ceaf8bec05..a16611f9d9 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -49,7 +49,9 @@ use illumos_utils::dladm::{ Dladm, Etherstub, EtherstubVnic, GetSimnetError, PhysicalLink, }; use illumos_utils::link::{Link, VnicAllocator}; -use illumos_utils::opte::{DhcpCfg, Port, PortCreateParams, PortManager, PortTicket}; +use illumos_utils::opte::{ + DhcpCfg, Port, PortCreateParams, PortManager, PortTicket, +}; use illumos_utils::running_zone::{ EnsureAddressError, InstalledZone, RunCommandError, RunningZone, ZoneBuilderFactory, diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 263db9555e..6f73d88fe6 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -51,7 +51,7 @@ use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; use omicron_common::api::internal::shared::{ - HostPortConfig, RackNetworkConfig, ReifiedVpcRouteSet, + HostPortConfig, RackNetworkConfig, ReifiedVpcRouteSet, ReifiedVpcRouteState, }; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, @@ -1091,7 +1091,7 @@ impl SledAgent { self.inner.bootstore.clone() } - pub fn list_vpc_routes(&self) -> Vec { + pub fn list_vpc_routes(&self) -> Vec { self.inner.port_manager.vpc_routes_list() } From 006b1ca3b278b33482d06f2ada784df02cdcf11f Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 21 May 2024 11:39:07 +0100 Subject: [PATCH 16/59] Iterating. --- clients/sled-agent-client/src/lib.rs | 1 + nexus/db-queries/src/db/datastore/vpc.rs | 52 ++++++++++++++++++++++++ nexus/src/app/background/vpc_routes.rs | 11 +++-- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 58a1fc0d14..9df5230779 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -56,6 +56,7 @@ progenitor::generate_api!( ReifiedVpcRoute = omicron_common::api::internal::shared::ReifiedVpcRoute, ReifiedVpcRouteSet = omicron_common::api::internal::shared::ReifiedVpcRouteSet, RouterTarget = omicron_common::api::internal::shared::RouterTarget, + RouterVersion = omicron_common::api::internal::shared::RouterVersion, SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig, Vni = omicron_common::api::external::Vni, NetworkInterface = omicron_common::api::internal::shared::NetworkInterface, diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 69972a701c..d6b9645f93 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1684,6 +1684,58 @@ impl DataStore { Ok(out) } + + /// Trigger an RPW version bump on a single VPC router in response + /// to CRUD operations on individual routes. + pub async fn vpc_router_increment_rpw_version( + &self, + opctx: &OpContext, + authz_router: &authz::VpcRouter, + ) -> UpdateResult<()> { + opctx.authorize(authz::Action::Modify, authz_router).await?; + + use db::schema::vpc_router::dsl; + diesel::update(dsl::vpc_router) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_router.id())) + .set(dsl::resolved_version.eq(dsl::resolved_version + 1)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_router), + ) + })?; + + Ok(()) + } + + /// Trigger an RPW version bump on all routers within a VPC in + /// response to changes to named entities (e.g., subnets, instances). + pub async fn vpc_increment_rpw_version( + &self, + opctx: &OpContext, + authz_vpc: &authz::Vpc, + ) -> UpdateResult<()> { + opctx.authorize(authz::Action::CreateChild, authz_vpc).await?; + + use db::schema::vpc_router::dsl; + diesel::update(dsl::vpc_router) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::vpc_id.eq(authz_vpc.id())) + .set(dsl::resolved_version.eq(dsl::resolved_version + 1)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_vpc), + ) + })?; + + Ok(()) + } } #[cfg(test)] diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index d4d6478c93..eb19753c27 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -192,7 +192,7 @@ impl BackgroundTask for VpcRouteManager { let Some(db_router) = db_routers.get(&set.id) else { // The sled wants to know about rules for a VPC // subnet with no custom router set. Send them - // the empty list, unset its table version. + // the empty list, and unset its table version. set_rules(set.id, None, HashSet::new()); continue; }; @@ -203,12 +203,11 @@ impl BackgroundTask for VpcRouteManager { router_id, }; - // Only attempt to resolve/push a ruleset if we have a different - // router ID than the sled, or a higher version number. + // Only attempt to resolve/push a ruleset if we have a + // different router ID than the sled, or a higher version + // number. match &set.version { - Some(v) - if v.router_id == router_id - && v.generation >= version.generation => + Some(v) if !v.is_replaced_by(&version) => { continue; } From c7de875d028d1e4bd19c602e20d7a97f643bf221 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 00:43:37 +0100 Subject: [PATCH 17/59] Trigger RPW in all the right spots. --- clients/sled-agent-client/src/lib.rs | 2 +- .../src/db/datastore/network_interface.rs | 20 +++++- nexus/db-queries/src/db/datastore/vpc.rs | 35 ++++------ nexus/src/app/background/vpc_routes.rs | 23 +++---- nexus/src/app/instance.rs | 1 + nexus/src/app/vpc_router.rs | 64 ++++++++++++++++--- nexus/src/app/vpc_subnet.rs | 31 ++++++--- openapi/sled-agent.json | 10 ++- sled-agent/src/probe_manager.rs | 28 +++++++- 9 files changed, 155 insertions(+), 59 deletions(-) diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 9df5230779..d4fb36004f 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -35,7 +35,7 @@ progenitor::generate_api!( PortConfigV1 = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, RouteConfig = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, IpNet = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, - OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] } + OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] }, RouterId = { derives = [PartialEq, Eq, Hash, Debug, Deserialize, Serialize] }, }, //TODO trade the manual transformations later in this file for the diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index b426c8b472..2ac1f531a3 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -137,11 +137,27 @@ impl DataStore { ), )); } - self.create_network_interface_raw(opctx, interface) + + let out = self + .create_network_interface_raw(opctx, interface) .await // Convert to `InstanceNetworkInterface` before returning; we know // this is valid as we've checked the condition on-entry. - .map(NetworkInterface::as_instance) + .map(NetworkInterface::as_instance)?; + + // `instance:xxx` targets in router rules resolve to the primary + // NIC of that instance. Accordingly, NIC create may cause dangling + // entries to re-resolve to a valid instance (even if it is not yet + // started). + // This will not trigger the RPW directly, we still need to do so + // in e.g. the instance watcher task. + if out.primary { + self.vpc_increment_rpw_version(opctx, out.vpc_id) + .await + .map_err(|e| network_interface::InsertError::External(e))?; + } + + Ok(out) } /// List network interfaces associated with a given service. diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index d6b9645f93..fd90c8a7e6 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1409,7 +1409,9 @@ impl DataStore { Ok(()) }}).await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + self.vpc_increment_rpw_version(opctx, vpc_id).await } /// Look up a VPC by VNI. @@ -1690,49 +1692,40 @@ impl DataStore { pub async fn vpc_router_increment_rpw_version( &self, opctx: &OpContext, - authz_router: &authz::VpcRouter, + router_id: Uuid, ) -> UpdateResult<()> { - opctx.authorize(authz::Action::Modify, authz_router).await?; + // NOTE: this operation and `vpc_increment_rpw_version` do not + // have auth checks, as these can occur in connection with unrelated + // resources -- the current user may have access to those, but be unable + // to modify the entire set of VPC routers in a project. use db::schema::vpc_router::dsl; diesel::update(dsl::vpc_router) .filter(dsl::time_deleted.is_null()) - .filter(dsl::id.eq(authz_router.id())) + .filter(dsl::id.eq(router_id)) .set(dsl::resolved_version.eq(dsl::resolved_version + 1)) .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_router), - ) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; Ok(()) } - /// Trigger an RPW version bump on all routers within a VPC in + /// Trigger an RPW version bump on *all* routers within a VPC in /// response to changes to named entities (e.g., subnets, instances). pub async fn vpc_increment_rpw_version( &self, opctx: &OpContext, - authz_vpc: &authz::Vpc, + vpc_id: Uuid, ) -> UpdateResult<()> { - opctx.authorize(authz::Action::CreateChild, authz_vpc).await?; - use db::schema::vpc_router::dsl; diesel::update(dsl::vpc_router) .filter(dsl::time_deleted.is_null()) - .filter(dsl::vpc_id.eq(authz_vpc.id())) + .filter(dsl::vpc_id.eq(vpc_id)) .set(dsl::resolved_version.eq(dsl::resolved_version + 1)) .execute_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_vpc), - ) - })?; + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; Ok(()) } diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index eb19753c27..6cdf720ce2 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -207,8 +207,7 @@ impl BackgroundTask for VpcRouteManager { // different router ID than the sled, or a higher version // number. match &set.version { - Some(v) if !v.is_replaced_by(&version) => - { + Some(v) if !v.is_replaced_by(&version) => { continue; } _ => {} @@ -244,15 +243,17 @@ impl BackgroundTask for VpcRouteManager { } } - if let Err(e) = client.set_vpc_routes(&to_push).await { - error!( - log, - "failed to push new VPC route state from sled"; - "sled" => sled.serial_number(), - "err" => ?e - ); - continue; - }; + if !to_push.is_empty() { + if let Err(e) = client.set_vpc_routes(&to_push).await { + error!( + log, + "failed to push new VPC route state from sled"; + "sled" => sled.serial_number(), + "err" => ?e + ); + continue; + }; + } } json!({}) diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 50b46c8e8d..090caddf18 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1524,6 +1524,7 @@ impl super::Nexus { new_runtime_state, ) .await?; + self.vpc_needed_notify_sleds(); Ok(()) } diff --git a/nexus/src/app/vpc_router.rs b/nexus/src/app/vpc_router.rs index e65b2a8605..ae2fdffeeb 100644 --- a/nexus/src/app/vpc_router.rs +++ b/nexus/src/app/vpc_router.rs @@ -83,6 +83,10 @@ impl super::Nexus { .db_datastore .vpc_create_router(&opctx, &authz_vpc, router) .await?; + + // Note: we don't trigger the route RPW here as it's impossible + // for the router to be bound to a subnet at this point. + Ok(router) } @@ -114,8 +118,8 @@ impl super::Nexus { .await } - // TODO: When a router is deleted it should be unassociated w/ any subnets it may be associated with - // or trigger an error + // TODO(now): When a router is deleted it should be unassociated w/ any subnets it may be associated with + // or trigger an error. pub(crate) async fn vpc_delete_router( &self, opctx: &OpContext, @@ -130,7 +134,12 @@ impl super::Nexus { if db_router.kind == VpcRouterKind::System { return Err(Error::invalid_request("Cannot delete system router")); } - self.db_datastore.vpc_delete_router(opctx, &authz_router).await + let out = + self.db_datastore.vpc_delete_router(opctx, &authz_router).await?; + + self.vpc_needed_notify_sleds(); + + Ok(out) } // Routes @@ -197,6 +206,9 @@ impl super::Nexus { .db_datastore .router_create_route(&opctx, &authz_router, route) .await?; + + self.vpc_router_increment_rpw_version(opctx, &authz_router).await?; + Ok(route) } @@ -219,7 +231,7 @@ impl super::Nexus { route_lookup: &lookup::RouterRoute<'_>, params: ¶ms::RouterRouteUpdate, ) -> UpdateResult { - let (.., vpc, _, authz_route, db_route) = + let (.., vpc, authz_router, authz_route, db_route) = route_lookup.fetch_for(authz::Action::Modify).await?; // TODO: Write a test for this once there's a way to test it (i.e. // subnets automatically register to the system router table) @@ -234,9 +246,14 @@ impl super::Nexus { ))); } } - self.db_datastore + let out = self + .db_datastore .router_update_route(&opctx, &authz_route, params.clone().into()) - .await + .await?; + + self.vpc_router_increment_rpw_version(opctx, &authz_router).await?; + + Ok(out) } pub(crate) async fn router_delete_route( @@ -244,7 +261,7 @@ impl super::Nexus { opctx: &OpContext, route_lookup: &lookup::RouterRoute<'_>, ) -> DeleteResult { - let (.., authz_route, db_route) = + let (.., authz_router, authz_route, db_route) = route_lookup.fetch_for(authz::Action::Delete).await?; // Only custom routes can be deleted @@ -254,6 +271,37 @@ impl super::Nexus { "DELETE not allowed on system routes", )); } - self.db_datastore.router_delete_route(opctx, &authz_route).await + let out = + self.db_datastore.router_delete_route(opctx, &authz_route).await?; + + self.vpc_router_increment_rpw_version(opctx, &authz_router).await?; + + Ok(out) + } + + /// Trigger the VPC routing RPW in repsonse to a state change + /// or a new possible listener (e.g., instance/probe start, NIC + /// create). + pub(crate) fn vpc_needed_notify_sleds(&self) { + self.background_tasks + .activate(&self.background_tasks.task_vpc_route_manager) + } + + /// Trigger an RPW version bump on a single VPC router in response + /// to CRUD operations on individual routes. + /// + /// This will also awaken the VPC Router RPW. + pub(crate) async fn vpc_router_increment_rpw_version( + &self, + opctx: &OpContext, + authz_router: &authz::VpcRouter, + ) -> UpdateResult<()> { + self.datastore() + .vpc_router_increment_rpw_version(opctx, authz_router.id()) + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(()) } } diff --git a/nexus/src/app/vpc_subnet.rs b/nexus/src/app/vpc_subnet.rs index 4c5a569201..0e3affb470 100644 --- a/nexus/src/app/vpc_subnet.rs +++ b/nexus/src/app/vpc_subnet.rs @@ -63,8 +63,7 @@ impl super::Nexus { )), } } - // TODO: When a subnet is created it should add a route entry into the VPC's - // system router + pub(crate) async fn vpc_create_subnet( &self, opctx: &OpContext, @@ -108,7 +107,7 @@ impl super::Nexus { // See for // details. let subnet_id = Uuid::new_v4(); - match params.ipv6_block { + let out = match params.ipv6_block { None => { const NUM_RETRIES: usize = 2; let mut retry = 0; @@ -212,7 +211,11 @@ impl super::Nexus { .map(|(.., subnet)| subnet) .map_err(SubnetError::into_external) } - } + }?; + + self.vpc_needed_notify_sleds(); + + Ok(out) } pub(crate) async fn vpc_subnet_list( @@ -234,13 +237,16 @@ impl super::Nexus { ) -> UpdateResult { let (.., authz_subnet) = vpc_subnet_lookup.lookup_for(authz::Action::Modify).await?; - self.db_datastore + let out = self + .db_datastore .vpc_update_subnet(&opctx, &authz_subnet, params.clone().into()) - .await + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(out) } - // TODO: When a subnet is deleted it should remove its entry from the VPC's - // system router. pub(crate) async fn vpc_delete_subnet( &self, opctx: &OpContext, @@ -248,9 +254,14 @@ impl super::Nexus { ) -> DeleteResult { let (.., authz_subnet, db_subnet) = vpc_subnet_lookup.fetch_for(authz::Action::Delete).await?; - self.db_datastore + let out = self + .db_datastore .vpc_delete_subnet(opctx, &db_subnet, &authz_subnet) - .await + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(out) } pub(crate) async fn subnet_list_instance_network_interfaces( diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index e4778bf06a..8c71d9eeb1 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -4340,13 +4340,17 @@ "uniqueItems": true }, "version": { - "$ref": "#/components/schemas/RouterVersion" + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RouterVersion" + } + ] } }, "required": [ "id", - "routes", - "version" + "routes" ] }, "ReifiedVpcRouteState": { diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index cc38725b61..eabf3850af 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -6,7 +6,9 @@ use illumos_utils::opte::params::VpcFirewallRule; use illumos_utils::opte::{DhcpCfg, PortCreateParams, PortManager}; use illumos_utils::running_zone::{RunningZone, ZoneBuilderFactory}; use illumos_utils::zone::Zones; -use nexus_client::types::{ProbeExternalIp, ProbeInfo}; +use nexus_client::types::{ + BackgroundTasksActivateRequest, ProbeExternalIp, ProbeInfo, +}; use omicron_common::api::external::{ VpcFirewallRuleAction, VpcFirewallRuleDirection, VpcFirewallRulePriority, VpcFirewallRuleStatus, @@ -179,24 +181,44 @@ impl ProbeManagerInner { } }; - self.add(target.difference(¤t)).await; + let n_added = self.add(target.difference(¤t)).await; self.remove(current.difference(&target)).await; self.check(current.intersection(&target)).await; + + // If we have created some new probes, we may (in future) need the control plane + // to provide us with valid routes for the VPC the probe belongs to. + if n_added > 0 { + if let Err(e) = self + .nexus_client + .client() + .bgtask_activate(&BackgroundTasksActivateRequest { + bgtask_names: vec!["vpc_route_manager".into()], + }) + .await + { + error!(self.log, "get routes for probe: {e}"); + } + } } }) } /// Add a set of probes to this sled. - async fn add<'a, I>(self: &Arc, probes: I) + /// + /// Returns the number of inserted probes. + async fn add<'a, I>(self: &Arc, probes: I) -> usize where I: Iterator, { + let mut i = 0; for probe in probes { info!(self.log, "adding probe {}", probe.id); if let Err(e) = self.add_probe(probe).await { error!(self.log, "add probe: {e}"); } + i += 1; } + i } /// Add a probe to this sled. This sets up resources for the probe zone From 40edbc8faf85a79a864a3b18d5c7e304b293f846 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 11:12:20 +0100 Subject: [PATCH 18/59] Unpublish VPC routers API. We'll get to that in the next PR. --- nexus/src/external_api/http_entrypoints.rs | 10 + nexus/tests/output/nexus_tags.txt | 10 - openapi/nexus.json | 1663 ++++---------------- 3 files changed, 289 insertions(+), 1394 deletions(-) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 2678768b48..e814df2b61 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -5446,6 +5446,7 @@ async fn vpc_firewall_rules_update( method = GET, path = "/v1/vpc-routers", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_list( rqctx: RequestContext, @@ -5485,6 +5486,7 @@ async fn vpc_router_list( method = GET, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_view( rqctx: RequestContext, @@ -5518,6 +5520,7 @@ async fn vpc_router_view( method = POST, path = "/v1/vpc-routers", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_create( rqctx: RequestContext, @@ -5553,6 +5556,7 @@ async fn vpc_router_create( method = DELETE, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_delete( rqctx: RequestContext, @@ -5586,6 +5590,7 @@ async fn vpc_router_delete( method = PUT, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_update( rqctx: RequestContext, @@ -5625,6 +5630,7 @@ async fn vpc_router_update( method = GET, path = "/v1/vpc-router-routes", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_list( rqctx: RequestContext, @@ -5666,6 +5672,7 @@ async fn vpc_router_route_list( method = GET, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_view( rqctx: RequestContext, @@ -5702,6 +5709,7 @@ async fn vpc_router_route_view( method = POST, path = "/v1/vpc-router-routes", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_create( rqctx: RequestContext, @@ -5737,6 +5745,7 @@ async fn vpc_router_route_create( method = DELETE, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_delete( rqctx: RequestContext, @@ -5772,6 +5781,7 @@ async fn vpc_router_route_delete( method = PUT, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_update( rqctx: RequestContext, diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 35d8c32561..a32fe5c4b9 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -232,16 +232,6 @@ vpc_delete DELETE /v1/vpcs/{vpc} vpc_firewall_rules_update PUT /v1/vpc-firewall-rules vpc_firewall_rules_view GET /v1/vpc-firewall-rules vpc_list GET /v1/vpcs -vpc_router_create POST /v1/vpc-routers -vpc_router_delete DELETE /v1/vpc-routers/{router} -vpc_router_list GET /v1/vpc-routers -vpc_router_route_create POST /v1/vpc-router-routes -vpc_router_route_delete DELETE /v1/vpc-router-routes/{route} -vpc_router_route_list GET /v1/vpc-router-routes -vpc_router_route_update PUT /v1/vpc-router-routes/{route} -vpc_router_route_view GET /v1/vpc-router-routes/{route} -vpc_router_update PUT /v1/vpc-routers/{router} -vpc_router_view GET /v1/vpc-routers/{router} vpc_subnet_create POST /v1/vpc-subnets vpc_subnet_delete DELETE /v1/vpc-subnets/{subnet} vpc_subnet_list GET /v1/vpc-subnets diff --git a/openapi/nexus.json b/openapi/nexus.json index 55f83f4a24..92af2a6b74 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -8346,14 +8346,13 @@ } } }, - "/v1/vpc-router-routes": { + "/v1/vpc-subnets": { "get": { "tags": [ "vpcs" ], - "summary": "List routes", - "description": "List the routes associated with a router in a particular VPC.", - "operationId": "vpc_router_route_list", + "summary": "List subnets", + "operationId": "vpc_subnet_list", "parameters": [ { "in": "query", @@ -8383,14 +8382,6 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "sort_by", @@ -8401,7 +8392,7 @@ { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8413,7 +8404,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteResultsPage" + "$ref": "#/components/schemas/VpcSubnetResultsPage" } } } @@ -8427,7 +8418,7 @@ }, "x-dropshot-pagination": { "required": [ - "router" + "vpc" ] } }, @@ -8435,8 +8426,8 @@ "tags": [ "vpcs" ], - "summary": "Create route", - "operationId": "vpc_router_route_create", + "summary": "Create subnet", + "operationId": "vpc_subnet_create", "parameters": [ { "in": "query", @@ -8446,19 +8437,11 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8468,7 +8451,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteCreate" + "$ref": "#/components/schemas/VpcSubnetCreate" } } }, @@ -8480,7 +8463,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -8494,18 +8477,18 @@ } } }, - "/v1/vpc-router-routes/{route}": { + "/v1/vpc-subnets/{subnet}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch route", - "operationId": "vpc_router_route_view", + "summary": "Fetch subnet", + "operationId": "vpc_subnet_view", "parameters": [ { "in": "path", - "name": "route", - "description": "Name or ID of the route", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8519,19 +8502,10 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8543,7 +8517,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -8560,13 +8534,13 @@ "tags": [ "vpcs" ], - "summary": "Update route", - "operationId": "vpc_router_route_update", + "summary": "Update subnet", + "operationId": "vpc_subnet_update", "parameters": [ { "in": "path", - "name": "route", - "description": "Name or ID of the route", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8580,18 +8554,10 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8601,7 +8567,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteUpdate" + "$ref": "#/components/schemas/VpcSubnetUpdate" } } }, @@ -8613,7 +8579,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -8630,13 +8596,13 @@ "tags": [ "vpcs" ], - "summary": "Delete route", - "operationId": "vpc_router_route_delete", + "summary": "Delete subnet", + "operationId": "vpc_subnet_delete", "parameters": [ { "in": "path", - "name": "route", - "description": "Name or ID of the route", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8650,18 +8616,10 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8680,14 +8638,23 @@ } } }, - "/v1/vpc-routers": { + "/v1/vpc-subnets/{subnet}/network-interfaces": { "get": { "tags": [ "vpcs" ], - "summary": "List routers", - "operationId": "vpc_router_list", + "summary": "List network interfaces", + "operationId": "vpc_subnet_list_network_interfaces", "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "limit", @@ -8738,7 +8705,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterResultsPage" + "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" } } } @@ -8751,30 +8718,89 @@ } }, "x-dropshot-pagination": { - "required": [ - "vpc" - ] + "required": [] } - }, - "post": { + } + }, + "/v1/vpcs": { + "get": { "tags": [ "vpcs" ], - "summary": "Create VPC router", - "operationId": "vpc_router_create", + "summary": "List VPCs", + "operationId": "vpc_list", "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create VPC", + "operationId": "vpc_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8785,7 +8811,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterCreate" + "$ref": "#/components/schemas/VpcCreate" } } }, @@ -8797,7 +8823,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/Vpc" } } } @@ -8811,18 +8837,18 @@ } } }, - "/v1/vpc-routers/{router}": { + "/v1/vpcs/{vpc}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch router", - "operationId": "vpc_router_view", + "summary": "Fetch VPC", + "operationId": "vpc_view", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8831,15 +8857,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8851,7 +8869,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/Vpc" } } } @@ -8868,13 +8886,13 @@ "tags": [ "vpcs" ], - "summary": "Update router", - "operationId": "vpc_router_update", + "summary": "Update a VPC", + "operationId": "vpc_update", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8883,15 +8901,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8901,7 +8911,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterUpdate" + "$ref": "#/components/schemas/VpcUpdate" } } }, @@ -8913,7 +8923,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/Vpc" } } } @@ -8930,13 +8940,13 @@ "tags": [ "vpcs" ], - "summary": "Delete router", - "operationId": "vpc_router_delete", + "summary": "Delete VPC", + "operationId": "vpc_delete", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8945,15 +8955,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8971,649 +8973,21 @@ } } } - }, - "/v1/vpc-subnets": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List subnets", - "operationId": "vpc_subnet_list", - "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnetResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "vpc" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create subnet", - "operationId": "vpc_subnet_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnetCreate" - } - } - }, - "required": true - }, - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnet" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/vpc-subnets/{subnet}": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "Fetch subnet", - "operationId": "vpc_subnet_view", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnet" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "tags": [ - "vpcs" - ], - "summary": "Update subnet", - "operationId": "vpc_subnet_update", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnetUpdate" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnet" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "vpcs" - ], - "summary": "Delete subnet", - "operationId": "vpc_subnet_delete", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/vpc-subnets/{subnet}/network-interfaces": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List network interfaces", - "operationId": "vpc_subnet_list_network_interfaces", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [] - } - } - }, - "/v1/vpcs": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List VPCs", - "operationId": "vpc_list", - "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "project" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create VPC", - "operationId": "vpc_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcCreate" - } - } - }, - "required": true - }, - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Vpc" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/vpcs/{vpc}": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "Fetch VPC", - "operationId": "vpc_view", - "parameters": [ - { - "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Vpc" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "tags": [ - "vpcs" - ], - "summary": "Update a VPC", - "operationId": "vpc_update", - "parameters": [ - { - "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcUpdate" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Vpc" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "vpcs" - ], - "summary": "Delete VPC", - "operationId": "vpc_delete", - "parameters": [ - { - "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - } - }, - "components": { - "schemas": { - "Address": { - "description": "An address tied to an address lot.", - "type": "object", - "properties": { - "address": { - "description": "The address and prefix length of this address.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] + } + }, + "components": { + "schemas": { + "Address": { + "description": "An address tied to an address lot.", + "type": "object", + "properties": { + "address": { + "description": "The address and prefix length of this address.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] }, "address_lot": { "description": "The address lot this address is drawn from.", @@ -16233,179 +15607,32 @@ }, "time_created": { "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - } - }, - "required": [ - "description", - "id", - "name", - "time_created", - "time_modified" - ] - }, - "ProjectCreate": { - "description": "Create-time parameters for a `Project`", - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "name": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "description", - "name" - ] - }, - "ProjectResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/Project" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" - ] - }, - "ProjectRole": { - "type": "string", - "enum": [ - "admin", - "collaborator", - "viewer" - ] - }, - "ProjectRolePolicy": { - "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", - "type": "object", - "properties": { - "role_assignments": { - "description": "Roles directly assigned on this resource", - "type": "array", - "items": { - "$ref": "#/components/schemas/ProjectRoleRoleAssignment" - } - } - }, - "required": [ - "role_assignments" - ] - }, - "ProjectRoleRoleAssignment": { - "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", - "type": "object", - "properties": { - "identity_id": { - "type": "string", - "format": "uuid" - }, - "identity_type": { - "$ref": "#/components/schemas/IdentityType" - }, - "role_name": { - "$ref": "#/components/schemas/ProjectRole" - } - }, - "required": [ - "identity_id", - "identity_type", - "role_name" - ] - }, - "ProjectUpdate": { - "description": "Updateable properties of a `Project`", - "type": "object", - "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "name": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - } - } - }, - "Rack": { - "description": "View of an Rack", - "type": "object", - "properties": { - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "time_created": { - "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - } - }, - "required": [ - "id", - "time_created", - "time_modified" - ] - }, - "RackResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/Rack" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" } }, "required": [ - "items" + "description", + "id", + "name", + "time_created", + "time_modified" ] }, - "Role": { - "description": "View of a Role", + "ProjectCreate": { + "description": "Create-time parameters for a `Project`", "type": "object", "properties": { "description": { "type": "string" }, "name": { - "$ref": "#/components/schemas/RoleName" + "$ref": "#/components/schemas/Name" } }, "required": [ @@ -16413,14 +15640,7 @@ "name" ] }, - "RoleName": { - "title": "A name for a built-in role", - "description": "Role names consist of two string components separated by dot (\".\").", - "type": "string", - "pattern": "[a-z-]+\\.[a-z-]+", - "maxLength": 63 - }, - "RoleResultsPage": { + "ProjectResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -16428,7 +15648,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/Role" + "$ref": "#/components/schemas/Project" } }, "next_page": { @@ -16441,284 +15661,77 @@ "items" ] }, - "Route": { - "description": "A route to a destination network through a gateway address.", - "type": "object", - "properties": { - "dst": { - "description": "The route destination.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] - }, - "gw": { - "description": "The route gateway.", - "type": "string", - "format": "ip" - }, - "vid": { - "nullable": true, - "description": "VLAN id the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "dst", - "gw" + "ProjectRole": { + "type": "string", + "enum": [ + "admin", + "collaborator", + "viewer" ] }, - "RouteConfig": { - "description": "Route configuration data associated with a switch port configuration.", + "ProjectRolePolicy": { + "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", "type": "object", "properties": { - "routes": { - "description": "The set of routes assigned to a switch port.", + "role_assignments": { + "description": "Roles directly assigned on this resource", "type": "array", "items": { - "$ref": "#/components/schemas/Route" + "$ref": "#/components/schemas/ProjectRoleRoleAssignment" } } }, "required": [ - "routes" - ] - }, - "RouteDestination": { - "description": "A `RouteDestination` is used to match traffic with a routing rule, on the destination of that traffic.\n\nWhen traffic is to be sent to a destination that is within a given `RouteDestination`, the corresponding `RouterRoute` applies, and traffic will be forward to the `RouteTarget` for that rule.", - "oneOf": [ - { - "description": "Route applies to traffic destined for a specific IP address", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ip" - ] - }, - "value": { - "type": "string", - "format": "ip" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Route applies to traffic destined for a specific IP subnet", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ip_net" - ] - }, - "value": { - "$ref": "#/components/schemas/IpNet" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Route applies to traffic destined for the given VPC.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "vpc" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Route applies to traffic", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "subnet" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - } + "role_assignments" ] }, - "RouteTarget": { - "description": "A `RouteTarget` describes the possible locations that traffic matching a route destination can be sent.", - "oneOf": [ - { - "description": "Forward traffic to a particular IP address.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ip" - ] - }, - "value": { - "type": "string", - "format": "ip" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to a VPC", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "vpc" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to a VPC Subnet", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "subnet" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to a specific instance", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "instance" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to an internet gateway", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "internet_gateway" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Drop matching traffic", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "drop" - ] - } - }, - "required": [ - "type" - ] + "ProjectRoleRoleAssignment": { + "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", + "type": "object", + "properties": { + "identity_id": { + "type": "string", + "format": "uuid" + }, + "identity_type": { + "$ref": "#/components/schemas/IdentityType" + }, + "role_name": { + "$ref": "#/components/schemas/ProjectRole" } + }, + "required": [ + "identity_id", + "identity_type", + "role_name" ] }, - "RouterRoute": { - "description": "A route defines a rule that governs where traffic should be sent based on its destination.", + "ProjectUpdate": { + "description": "Updateable properties of a `Project`", "type": "object", "properties": { "description": { - "description": "human-readable free-form text about a resource", + "nullable": true, "type": "string" }, - "destination": { - "$ref": "#/components/schemas/RouteDestination" - }, - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "kind": { - "description": "Describes the kind of router. Set at creation. `read-only`", - "allOf": [ - { - "$ref": "#/components/schemas/RouterRouteKind" - } - ] - }, "name": { - "description": "unique, mutable, user-controlled identifier for each resource", + "nullable": true, "allOf": [ { "$ref": "#/components/schemas/Name" } ] - }, - "target": { - "$ref": "#/components/schemas/RouteTarget" + } + } + }, + "Rack": { + "description": "View of an Rack", + "type": "object", + "properties": { + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" }, "time_created": { "description": "timestamp when this resource was created", @@ -16729,83 +15742,59 @@ "description": "timestamp when this resource was last modified", "type": "string", "format": "date-time" - }, - "vpc_router_id": { - "description": "The ID of the VPC Router to which the route belongs", - "type": "string", - "format": "uuid" } }, "required": [ - "description", - "destination", "id", - "kind", - "name", - "target", "time_created", - "time_modified", - "vpc_router_id" + "time_modified" + ] + }, + "RackResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Rack" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" ] }, - "RouterRouteCreate": { - "description": "Create-time parameters for a `RouterRoute`", + "Role": { + "description": "View of a Role", "type": "object", "properties": { "description": { "type": "string" }, - "destination": { - "$ref": "#/components/schemas/RouteDestination" - }, "name": { - "$ref": "#/components/schemas/Name" - }, - "target": { - "$ref": "#/components/schemas/RouteTarget" + "$ref": "#/components/schemas/RoleName" } }, "required": [ "description", - "destination", - "name", - "target" + "name" ] }, - "RouterRouteKind": { - "description": "The kind of a `RouterRoute`\n\nThe kind determines certain attributes such as if the route is modifiable and describes how or where the route was created.", - "oneOf": [ - { - "description": "Determines the default destination of traffic, such as whether it goes to the internet or not.\n\n`Destination: An Internet Gateway` `Modifiable: true`", - "type": "string", - "enum": [ - "default" - ] - }, - { - "description": "Automatically added for each VPC Subnet in the VPC\n\n`Destination: A VPC Subnet` `Modifiable: false`", - "type": "string", - "enum": [ - "vpc_subnet" - ] - }, - { - "description": "Automatically added when VPC peering is established\n\n`Destination: A different VPC` `Modifiable: false`", - "type": "string", - "enum": [ - "vpc_peering" - ] - }, - { - "description": "Created by a user; see `RouteTarget`\n\n`Destination: User defined` `Modifiable: true`", - "type": "string", - "enum": [ - "custom" - ] - } - ] + "RoleName": { + "title": "A name for a built-in role", + "description": "Role names consist of two string components separated by dot (\".\").", + "type": "string", + "pattern": "[a-z-]+\\.[a-z-]+", + "maxLength": 63 }, - "RouterRouteResultsPage": { + "RoleResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -16813,7 +15802,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/Role" } }, "next_page": { @@ -16826,32 +15815,50 @@ "items" ] }, - "RouterRouteUpdate": { - "description": "Updateable properties of a `RouterRoute`", + "Route": { + "description": "A route to a destination network through a gateway address.", "type": "object", "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "destination": { - "$ref": "#/components/schemas/RouteDestination" - }, - "name": { - "nullable": true, + "dst": { + "description": "The route destination.", "allOf": [ { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/IpNet" } ] }, - "target": { - "$ref": "#/components/schemas/RouteTarget" + "gw": { + "description": "The route gateway.", + "type": "string", + "format": "ip" + }, + "vid": { + "nullable": true, + "description": "VLAN id the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "dst", + "gw" + ] + }, + "RouteConfig": { + "description": "Route configuration data associated with a switch port configuration.", + "type": "object", + "properties": { + "routes": { + "description": "The set of routes assigned to a switch port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Route" + } } }, "required": [ - "destination", - "target" + "routes" ] }, "SamlIdentityProvider": { @@ -19949,118 +18956,6 @@ "items" ] }, - "VpcRouter": { - "description": "A VPC router defines a series of rules that indicate where traffic should be sent depending on its destination.", - "type": "object", - "properties": { - "description": { - "description": "human-readable free-form text about a resource", - "type": "string" - }, - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "kind": { - "$ref": "#/components/schemas/VpcRouterKind" - }, - "name": { - "description": "unique, mutable, user-controlled identifier for each resource", - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - }, - "time_created": { - "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - }, - "vpc_id": { - "description": "The VPC to which the router belongs.", - "type": "string", - "format": "uuid" - } - }, - "required": [ - "description", - "id", - "kind", - "name", - "time_created", - "time_modified", - "vpc_id" - ] - }, - "VpcRouterCreate": { - "description": "Create-time parameters for a `VpcRouter`", - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "name": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "description", - "name" - ] - }, - "VpcRouterKind": { - "type": "string", - "enum": [ - "system", - "custom" - ] - }, - "VpcRouterResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/VpcRouter" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" - ] - }, - "VpcRouterUpdate": { - "description": "Updateable properties of a `VpcRouter`", - "type": "object", - "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "name": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - } - } - }, "VpcSubnet": { "description": "A VPC subnet represents a logical grouping for instances that allows network traffic between them, within a IPv4 subnetwork or optionally an IPv6 subnetwork.", "type": "object", From 3b3abb1d1b89adef7d9a1202aacac607e320e4d9 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 16:45:06 +0100 Subject: [PATCH 19/59] Fixup broken tests. --- dev-tools/omdb/tests/env.out | 12 ++++ dev-tools/omdb/tests/successes.out | 13 +++- dev-tools/preprocessed_configs/config.xml | 41 ++++++++++++ nexus/src/app/sagas/vpc_create.rs | 17 ++++- .../tests/integration_tests/router_routes.rs | 65 +++++++++++++------ 5 files changed, 124 insertions(+), 24 deletions(-) create mode 100644 dev-tools/preprocessed_configs/config.xml diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 5716510602..b36280980e 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -114,6 +114,10 @@ task: "switch_port_config_manager" manages switch port settings for rack switches +task: "vpc_route_manager" + propagates updated VPC routes to all OPTE ports + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT @@ -225,6 +229,10 @@ task: "switch_port_config_manager" manages switch port settings for rack switches +task: "vpc_route_manager" + propagates updated VPC routes to all OPTE ports + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. @@ -323,6 +331,10 @@ task: "switch_port_config_manager" manages switch port settings for rack switches +task: "vpc_route_manager" + propagates updated VPC routes to all OPTE ports + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index c4c28460b8..07960112d6 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -291,6 +291,10 @@ task: "switch_port_config_manager" manages switch port settings for rack switches +task: "vpc_route_manager" + propagates updated VPC routes to all OPTE ports + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ @@ -426,7 +430,7 @@ task: "metrics_producer_gc" currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms -warning: unknown background task: "metrics_producer_gc" (don't know how to interpret details: Object {"expiration": String(""), "pruned": Array []}) +warning: unknown background task: "metrics_producer_gc" (don't know how to interpret details: Object {"expiration": String(""), "pruned": Array []}) task: "phantom_disks" configured period: every 30s @@ -471,6 +475,13 @@ task: "switch_port_config_manager" started at (s ago) and ran for ms warning: unknown background task: "switch_port_config_manager" (don't know how to interpret details: Object {}) +task: "vpc_route_manager" + configured period: every 30s + currently executing: no + last completed activation: , triggered by an explicit signal + started at (s ago) and ran for ms +warning: unknown background task: "vpc_route_manager" (don't know how to interpret details: Object {}) + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ diff --git a/dev-tools/preprocessed_configs/config.xml b/dev-tools/preprocessed_configs/config.xml new file mode 100644 index 0000000000..9b13f12aea --- /dev/null +++ b/dev-tools/preprocessed_configs/config.xml @@ -0,0 +1,41 @@ + + + + + trace + true + + + 8123 + 9000 + 9004 + + ./ + + true + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + \ No newline at end of file diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index cc62d9315d..c84cf5ff20 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -587,12 +587,25 @@ pub(crate) mod test { .await .expect("Failed to delete default Subnet"); - // Default route + // Default gateway routes let (.., authz_route, _route) = LookupPath::new(&opctx, &datastore) .project_id(project_id) .vpc_name(&default_name.clone().into()) .vpc_router_name(&system_name.clone().into()) - .router_route_name(&default_name.clone().into()) + .router_route_name(&"default-v4".parse::().unwrap().into()) + .fetch() + .await + .expect("Failed to fetch default route"); + datastore + .router_delete_route(&opctx, &authz_route) + .await + .expect("Failed to delete default route"); + + let (.., authz_route, _route) = LookupPath::new(&opctx, &datastore) + .project_id(project_id) + .vpc_name(&default_name.clone().into()) + .vpc_router_name(&system_name.clone().into()) + .router_route_name(&"default-v6".parse::().unwrap().into()) .fetch() .await .expect("Failed to fetch default route"); diff --git a/nexus/tests/integration_tests/router_routes.rs b/nexus/tests/integration_tests/router_routes.rs index 10c594bba9..a13026a7fd 100644 --- a/nexus/tests/integration_tests/router_routes.rs +++ b/nexus/tests/integration_tests/router_routes.rs @@ -10,6 +10,8 @@ use nexus_test_utils::identity_eq; use nexus_test_utils::resource_helpers::objects_list_page_authz; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; +use omicron_common::api::external::IpNet; +use omicron_common::api::external::SimpleIdentity; use omicron_common::api::external::{ IdentityMetadataCreateParams, IdentityMetadataUpdateParams, RouteDestination, RouteTarget, RouterRoute, RouterRouteKind, @@ -59,27 +61,48 @@ async fn test_router_routes(cptestctx: &ControlPlaneTestContext) { .await .items; - // The system should start with a single, pre-configured route - assert_eq!(system_router_routes.len(), 1); - - // That route should be the default route - let default_route = &system_router_routes[0]; - assert_eq!(default_route.kind, RouterRouteKind::Default); - - // It errors if you try to delete the default route - let error: dropshot::HttpErrorResponseBody = NexusRequest::expect_failure( - client, - StatusCode::BAD_REQUEST, - Method::DELETE, - get_route_url("system", "default").as_str(), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .unwrap() - .parsed_body() - .unwrap(); - assert_eq!(error.message, "DELETE not allowed on system routes"); + // The system should start with three preconfigured routes: + // - a default v4 gateway route + // - a default v6 gateway route + // - a managed subnet route for the 'default' subnet + assert_eq!(system_router_routes.len(), 3); + + let mut v4_route = None; + let mut v6_route = None; + let mut subnet_route = None; + for route in system_router_routes { + match (&route.kind, &route.destination, &route.target) { + (RouterRouteKind::Default, RouteDestination::IpNet(IpNet::V4(_)), RouteTarget::InternetGateway(_)) => {v4_route = Some(route);}, + (RouterRouteKind::Default, RouteDestination::IpNet(IpNet::V6(_)), RouteTarget::InternetGateway(_)) => {v6_route = Some(route);}, + (RouterRouteKind::VpcSubnet, RouteDestination::Subnet(n0), RouteTarget::Subnet(n1)) if n0 == n1 && n0.as_str() == "default" => {subnet_route = Some(route);}, + _ => panic!("unexpected system route {route:?} -- wanted gateway and subnet"), + } + } + + let v4_route = + v4_route.expect("no v4 gateway route found in system router"); + let v6_route = + v6_route.expect("no v6 gateway route found in system router"); + let subnet_route = + subnet_route.expect("no default subnet route found in system router"); + + // Deleting any default system route is disallowed. + for route in &[&v4_route, &v6_route, &subnet_route] { + let error: dropshot::HttpErrorResponseBody = + NexusRequest::expect_failure( + client, + StatusCode::BAD_REQUEST, + Method::DELETE, + get_route_url("system", route.name().as_str()).as_str(), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + assert_eq!(error.message, "DELETE not allowed on system routes"); + } // Create a custom router create_router(&client, project_name, vpc_name, router_name).await; From be9f8ab7be6795605c981c0b04e11187c600c197 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 16:46:00 +0100 Subject: [PATCH 20/59] Accidental local state... --- dev-tools/preprocessed_configs/config.xml | 41 ----------------------- 1 file changed, 41 deletions(-) delete mode 100644 dev-tools/preprocessed_configs/config.xml diff --git a/dev-tools/preprocessed_configs/config.xml b/dev-tools/preprocessed_configs/config.xml deleted file mode 100644 index 9b13f12aea..0000000000 --- a/dev-tools/preprocessed_configs/config.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - trace - true - - - 8123 - 9000 - 9004 - - ./ - - true - - - - - - - ::/0 - - - default - default - 1 - - - - - - - - - - - \ No newline at end of file From f433b38574e50b6759d132c9d27587c4e955b897 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 22 May 2024 18:49:50 +0100 Subject: [PATCH 21/59] Unsubscribe routes from sled when ports are removed. --- dev-tools/omdb/tests/successes.out | 2 +- illumos-utils/src/opte/port.rs | 12 +++++ illumos-utils/src/opte/port_manager.rs | 72 ++++++++++++++++++-------- 3 files changed, 62 insertions(+), 24 deletions(-) diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 07960112d6..b1476b5f37 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -430,7 +430,7 @@ task: "metrics_producer_gc" currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms -warning: unknown background task: "metrics_producer_gc" (don't know how to interpret details: Object {"expiration": String(""), "pruned": Array []}) +warning: unknown background task: "metrics_producer_gc" (don't know how to interpret details: Object {"expiration": String(""), "pruned": Array []}) task: "phantom_disks" configured period: every 30s diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 38ba1b8c1c..832335891c 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -7,7 +7,9 @@ use crate::opte::Gateway; use crate::opte::Vni; use macaddr::MacAddr6; +use omicron_common::api::external; use omicron_common::api::external::IpNet; +use omicron_common::api::internal::shared::RouterId; use std::net::IpAddr; use std::sync::Arc; @@ -134,4 +136,14 @@ impl Port { pub fn slot(&self) -> u8 { self.inner.slot } + + pub fn system_router_key(&self) -> RouterId { + // Unwrap safety: both of these VNI types represent validated u24s. + let vni = external::Vni::try_from(self.vni().as_u32()).unwrap(); + RouterId { vni, subnet: None } + } + + pub fn custom_router_key(&self) -> RouterId { + RouterId { subnet: Some(*self.subnet()), ..self.system_router_key() } + } } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index d206273c48..943e818832 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -60,6 +60,7 @@ const XDE_LINK_PREFIX: &str = "opte"; struct RouteSet { version: Option, routes: HashSet, + active_ports: usize, } #[derive(Debug)] @@ -76,9 +77,6 @@ struct PortManagerInner { // (which includes the Uuid of the parent instance or service) ports: Mutex>, - // XX: Should this be the UUID of the VPC? The rulesets are - // arguably shared v4+v6, although today we don't yet - // allow dual-stack, let alone v6. // Map of all current resolved routes. routes: Mutex>, } @@ -380,12 +378,9 @@ impl PortManager { (port, ticket) }; - // XX: need to delete safely after all subnet-holders leave - // to not get flooded with useless rules. let mut routes = self.inner.routes.lock().unwrap(); - let system_routes = routes - .entry(RouterId { vni: nic.vni, subnet: None }) - .or_insert_with(|| { + let system_routes = + routes.entry(port.system_router_key()).or_insert_with(|| { let mut routes = HashSet::new(); // Services do not talk to one another via OPTE, but do need @@ -402,16 +397,20 @@ impl PortManager { }); } - RouteSet { version: None, routes } - }) - .clone(); + RouteSet { version: None, routes, active_ports: 0 } + }); + system_routes.active_ports += 1; + // Needed to get borrowck on our side, sadly. + let system_routes = system_routes.clone(); let custom_routes = routes - .entry(RouterId { vni: nic.vni, subnet: Some(nic.subnet) }) + .entry(port.custom_router_key()) .or_insert_with(|| RouteSet { version: None, routes: HashSet::default(), + active_ports: 0, }); + custom_routes.active_ports += 1; for (class, routes) in [ (RouterClass::System, &system_routes), @@ -460,11 +459,16 @@ impl PortManager { let mut routes = self.inner.routes.lock().unwrap(); let mut deltas = HashMap::new(); for set in new_routes { + // Disregard any route information for a subnet we don't have. + let Some(old) = routes.get(&set.id) else { + continue; + }; + // We have to handle subnet router changes, as well as // spurious updates from multiple Nexus instances. // If there's a UUID match, only update if vers increased, // otherwise take the update verbatim (including loss of version). - let (to_add, to_delete) = if let Some(old) = routes.get(&set.id) { + let (to_add, to_delete): (HashSet<_>, HashSet<_>) = match (old.version, set.version) { (Some(old_vers), Some(new_vers)) if !old_vers.is_replaced_by(&new_vers) => @@ -475,30 +479,29 @@ impl PortManager { set.routes.difference(&old.routes).cloned().collect(), old.routes.difference(&set.routes).cloned().collect(), ), - } - } else { - (set.routes.clone(), HashSet::new()) - }; + }; deltas.insert(set.id, (to_add, to_delete)); + let active_ports = old.active_ports; routes.insert( set.id, - RouteSet { version: set.version, routes: set.routes }, + RouteSet { + version: set.version, + routes: set.routes, + active_ports, + }, ); } - drop(routes); let ports = self.inner.ports.lock().unwrap(); #[cfg(target_os = "illumos")] let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?; for port in ports.values() { - let vni = external::Vni::try_from(port.vni().as_u32()).unwrap(); - let system_id = RouterId { vni, subnet: None }; + let system_id = port.system_router_key(); let system_delta = deltas.get(&system_id); - let custom_id = RouterId { vni, subnet: Some(*port.subnet()) }; - + let custom_id = port.custom_router_key(); let custom_delta = deltas.get(&custom_id); #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] @@ -821,6 +824,29 @@ impl PortTicket { ); return Err(Error::ReleaseMissingPort(self.id, self.kind)); }; + drop(ports); + + // Cleanup the set of subnets we want to receive routes for. + let mut routes = self.manager.routes.lock().unwrap(); + for key in [port.system_router_key(), port.custom_router_key()] { + let should_remove = routes + .get_mut(&key) + .map(|v| { + v.active_ports = v.active_ports.saturating_sub(1); + v.active_ports == 0 + }) + .unwrap_or_default(); + + if should_remove { + routes.remove(&key); + info!( + self.manager.log, + "Removed route set for subnet"; + "id" => ?&key, + ); + } + } + debug!( self.manager.log, "Removed OPTE port from manager"; From 62ca9f0bce8d053d4c6fb091672752e458612fbd Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 04:10:19 +0100 Subject: [PATCH 22/59] Migration query for subnet route creation. --- schema/crdb/vpc-subnet-routing/up03.sql | 108 ++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 schema/crdb/vpc-subnet-routing/up03.sql diff --git a/schema/crdb/vpc-subnet-routing/up03.sql b/schema/crdb/vpc-subnet-routing/up03.sql new file mode 100644 index 0000000000..9340de0c46 --- /dev/null +++ b/schema/crdb/vpc-subnet-routing/up03.sql @@ -0,0 +1,108 @@ +-- We need to manually rebuild a compliant set of routes. +-- Remove everything that exists today. +DELETE FROM omicron.public.router_route WHERE 1=1; + +-- Insert fixed_data routes for the services VPC. +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +VALUES + ( + '001de000-074c-4000-8000-000000000002', 'default-v4', + 'Default internet gateway route for Oxide Services', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'default', + 'inetgw:outbound', 'ipnet:0.0.0.0/0' + ), + ( + '001de000-074c-4000-8000-000000000003', 'default-v6', + 'Default internet gateway route for Oxide Services', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'default', + 'inetgw:outbound', 'ipnet:::/0' + ), + ( + '001de000-c470-4000-8000-000000000004', 'sn-external-dns', + 'Built-in VPC Subnet for Oxide service (external-dns)', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'vpc_subnet', + 'subnet:external-dns', 'subnet:external-dns' + ), + ( + '001de000-c470-4000-8000-000000000005', 'sn-nexus', + 'Built-in VPC Subnet for Oxide service (nexus)', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'vpc_subnet', + 'subnet:nexus', 'subnet:nexus' + ), + ( + '001de000-c470-4000-8000-000000000006', 'sn-boundary-ntp', + 'Built-in VPC Subnet for Oxide service (nexus)', + now(), now(), + '001de000-074c-4000-8000-000000000001', 'vpc_subnet', + 'subnet:boundary-ntp', 'subnet:boundary-ntp' + ) +ON CONFLICT DO NOTHING; + +-- Insert gateway routes for user VPCs. +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +SELECT + gen_random_uuid(), 'default-v4', + 'The default route of a vpc', + now(), now(), + omicron.public.vpc_router.id, 'default', + 'inetgw:outbound', 'ipnet:0.0.0.0/0' +FROM + omicron.public.vpc_router +ON CONFLICT DO NOTHING; + +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +SELECT + gen_random_uuid(), 'default-v6', + 'The default route of a vpc', + now(), now(), + omicron.public.vpc_router.id, 'default', + 'inetgw:outbound', 'ipnet:::/0' +FROM + omicron.public.vpc_router +ON CONFLICT DO NOTHING; + +-- Insert subnet routes for every defined VPC subnet. +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +SELECT + gen_random_uuid(), 'sn-' || vpc_subnet.name, + 'VPC Subnet route for ''' || vpc_subnet.name || '''', + now(), now(), + omicron.public.vpc_router.id, 'default', + 'subnet:' || vpc_subnet.name, 'subnet:' || vpc_subnet.name +FROM + (omicron.public.vpc_subnet JOIN omicron.public.vpc + ON vpc_subnet.vpc_id = vpc.id) JOIN omicron.public.vpc_router + ON vpc_router.vpc_id = vpc.id +ON CONFLICT DO NOTHING; From 2c06ff48b4684a96bf41073f7ef3f4ba42cff625 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 11:12:06 +0100 Subject: [PATCH 23/59] Rework migration slightly. --- schema/crdb/vpc-subnet-routing/up03.sql | 84 +++++++++++-------------- 1 file changed, 36 insertions(+), 48 deletions(-) diff --git a/schema/crdb/vpc-subnet-routing/up03.sql b/schema/crdb/vpc-subnet-routing/up03.sql index 9340de0c46..d256921d34 100644 --- a/schema/crdb/vpc-subnet-routing/up03.sql +++ b/schema/crdb/vpc-subnet-routing/up03.sql @@ -1,55 +1,10 @@ +set local disallow_full_table_scans = off; + -- We need to manually rebuild a compliant set of routes. -- Remove everything that exists today. DELETE FROM omicron.public.router_route WHERE 1=1; --- Insert fixed_data routes for the services VPC. -INSERT INTO omicron.public.router_route - ( - id, name, - description, - time_created, time_modified, - vpc_router_id, kind, - target, destination - ) -VALUES - ( - '001de000-074c-4000-8000-000000000002', 'default-v4', - 'Default internet gateway route for Oxide Services', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'default', - 'inetgw:outbound', 'ipnet:0.0.0.0/0' - ), - ( - '001de000-074c-4000-8000-000000000003', 'default-v6', - 'Default internet gateway route for Oxide Services', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'default', - 'inetgw:outbound', 'ipnet:::/0' - ), - ( - '001de000-c470-4000-8000-000000000004', 'sn-external-dns', - 'Built-in VPC Subnet for Oxide service (external-dns)', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'vpc_subnet', - 'subnet:external-dns', 'subnet:external-dns' - ), - ( - '001de000-c470-4000-8000-000000000005', 'sn-nexus', - 'Built-in VPC Subnet for Oxide service (nexus)', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'vpc_subnet', - 'subnet:nexus', 'subnet:nexus' - ), - ( - '001de000-c470-4000-8000-000000000006', 'sn-boundary-ntp', - 'Built-in VPC Subnet for Oxide service (nexus)', - now(), now(), - '001de000-074c-4000-8000-000000000001', 'vpc_subnet', - 'subnet:boundary-ntp', 'subnet:boundary-ntp' - ) -ON CONFLICT DO NOTHING; - --- Insert gateway routes for user VPCs. +-- Insert gateway routes for all VPCs. INSERT INTO omicron.public.router_route ( id, name, @@ -106,3 +61,36 @@ FROM ON vpc_subnet.vpc_id = vpc.id) JOIN omicron.public.vpc_router ON vpc_router.vpc_id = vpc.id ON CONFLICT DO NOTHING; + +-- Replace IDs of fixed_data routes for the services VPC. +-- This is done instead of an insert to match the initial +-- empty state of dbinit.sql. +WITH known_ids (new_id, new_name, new_description) AS ( + VALUES + ( + '001de000-074c-4000-8000-000000000002', 'default-v4', + 'Default internet gateway route for Oxide Services' + ), + ( + '001de000-074c-4000-8000-000000000003', 'default-v6', + 'Default internet gateway route for Oxide Services' + ), + ( + '001de000-c470-4000-8000-000000000004', 'sn-external-dns', + 'Built-in VPC Subnet for Oxide service (external-dns)' + ), + ( + '001de000-c470-4000-8000-000000000005', 'sn-nexus', + 'Built-in VPC Subnet for Oxide service (nexus)' + ), + ( + '001de000-c470-4000-8000-000000000006', 'sn-boundary-ntp', + 'Built-in VPC Subnet for Oxide service (boundary-ntp)' + ) +) +UPDATE omicron.public.router_route +SET + id = CAST(new_id AS UUID), + description = new_description +FROM known_ids +WHERE vpc_router_id = '001de000-074c-4000-8000-000000000001' AND new_name = router_route.name; From 0e8d1adf57f3041436deed9c5829e1a58068b0be Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 11:22:48 +0100 Subject: [PATCH 24/59] Bump OPTE to include latest perf work. --- Cargo.lock | 12 ++++++------ Cargo.toml | 4 ++-- tools/opte_version | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad3ac965d0..924cf4d0f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3470,7 +3470,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" [[package]] name = "illumos-utils" @@ -3883,7 +3883,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "quote", "syn 2.0.64", @@ -6008,7 +6008,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "cfg-if", "dyn-clone", @@ -6025,7 +6025,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -6037,7 +6037,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -6111,7 +6111,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=cf268a6383861f902b0c18afb7cfe35e42987504#cf268a6383861f902b0c18afb7cfe35e42987504" +source = "git+https://github.com/oxidecomputer/opte?rev=d6177ca84f23e60a661461bb4cece475689502d2#d6177ca84f23e60a661461bb4cece475689502d2" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 8d298d1feb..f12c6c72f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -347,14 +347,14 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "d6177ca84f23e60a661461bb4cece475689502d2", features = [ "api", "std" ] } once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "cf268a6383861f902b0c18afb7cfe35e42987504" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "d6177ca84f23e60a661461bb4cece475689502d2" } oso = "0.27" owo-colors = "4.0.0" oximeter = { path = "oximeter/oximeter" } diff --git a/tools/opte_version b/tools/opte_version index fc3e603c41..6126a52eb4 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.31.258 +0.31.259 From 25372224b295cbd4175252876de9809aef35a479 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 14:26:14 +0100 Subject: [PATCH 25/59] Self-review pt.1. --- clients/sled-agent-client/src/lib.rs | 5 +- common/src/api/internal/shared.rs | 27 ++++--- illumos-utils/src/opte/mod.rs | 2 + illumos-utils/src/opte/port_manager.rs | 57 +++++++++------ nexus/db-model/src/schema_versions.rs | 2 +- nexus/db-queries/src/db/datastore/instance.rs | 52 ------------- .../src/db/datastore/network_interface.rs | 4 +- nexus/db-queries/src/db/datastore/vpc.rs | 73 ++++++++++++++++--- nexus/db-queries/src/db/fixed_data/vpc.rs | 3 +- .../src/db/fixed_data/vpc_subnet.rs | 3 + nexus/src/app/background/vpc_routes.rs | 8 +- nexus/src/app/sagas/vpc_create.rs | 2 - nexus/src/app/vpc_router.rs | 2 - openapi/sled-agent.json | 38 +++++----- schema/crdb/dbinit.sql | 2 +- sled-agent/src/http_entrypoints.rs | 6 +- sled-agent/src/probe_manager.rs | 2 +- sled-agent/src/sled_agent.rs | 7 +- 18 files changed, 156 insertions(+), 139 deletions(-) diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index aaa59d0e98..4910918884 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -35,7 +35,6 @@ progenitor::generate_api!( PortConfigV1 = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, RouteConfig = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, IpNet = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, - RouterId = { derives = [PartialEq, Eq, Hash, Debug, Deserialize, Serialize] }, VirtualNetworkInterfaceHost = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] }, OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] }, }, @@ -54,8 +53,8 @@ progenitor::generate_api!( PortFec = omicron_common::api::internal::shared::PortFec, PortSpeed = omicron_common::api::internal::shared::PortSpeed, RouterId = omicron_common::api::internal::shared::RouterId, - ReifiedVpcRoute = omicron_common::api::internal::shared::ReifiedVpcRoute, - ReifiedVpcRouteSet = omicron_common::api::internal::shared::ReifiedVpcRouteSet, + ResolvedVpcRoute = omicron_common::api::internal::shared::ResolvedVpcRoute, + ResolvedVpcRouteSet = omicron_common::api::internal::shared::ResolvedVpcRouteSet, RouterTarget = omicron_common::api::internal::shared::RouterTarget, RouterVersion = omicron_common::api::internal::shared::RouterVersion, SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig, diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 52601a01a8..51b829e214 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -594,7 +594,7 @@ impl TryFrom<&[IpNetwork]> for IpAllowList { #[derive( Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] -pub struct ReifiedVpcRoute { +pub struct ResolvedVpcRoute { pub dest: IpNet, pub target: RouterTarget, } @@ -611,23 +611,28 @@ pub enum RouterTarget { VpcSubnet(IpNet), } -/// XXX +/// Information on the current parent router (and version) of a route set +/// according to the control plane. #[derive( Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] pub struct RouterVersion { pub router_id: Uuid, - pub generation: u64, + pub version: u64, } impl RouterVersion { + /// Return whether a new route set should be applied over the current + /// values. + /// + /// This will occur when seeing a new version and a matching parent, + /// or a new parent router on the control plane. pub fn is_replaced_by(&self, other: &Self) -> bool { - (self.router_id != other.router_id) - || self.generation < other.generation + (self.router_id != other.router_id) || self.version < other.version } } -/// Implementation details on XXX +/// Identifier for a VPC and/or subnet. #[derive( Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, )] @@ -636,19 +641,19 @@ pub struct RouterId { pub subnet: Option, } -/// Version information +/// Version information for routes on a given VPC subnet. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] -pub struct ReifiedVpcRouteState { +pub struct ResolvedVpcRouteState { pub id: RouterId, pub version: Option, } -/// An updated set of routes for a given +/// An updated set of routes for a given VPC and/or subnet. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] -pub struct ReifiedVpcRouteSet { +pub struct ResolvedVpcRouteSet { pub id: RouterId, pub version: Option, - pub routes: HashSet, + pub routes: HashSet, } #[cfg(test)] diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index f6ef186808..e53468f40e 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -73,6 +73,7 @@ impl Gateway { } } +/// Convert a nexus `IpNet` to an OPTE `IpCidr`. fn net_to_cidr(net: IpNet) -> IpCidr { match net { IpNet::V4(net) => IpCidr::Ip4(Ipv4Cidr::new( @@ -86,6 +87,7 @@ fn net_to_cidr(net: IpNet) -> IpCidr { } } +/// Convert a nexus `RouterTarget` to an OPTE `RouterTarget`. fn router_target_opte(target: &shared::RouterTarget) -> RouterTarget { use shared::RouterTarget::*; match target { diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 769634742b..caeda81217 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -16,9 +16,9 @@ use ipnetwork::IpNetwork; use omicron_common::api::external; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; -use omicron_common::api::internal::shared::ReifiedVpcRoute; -use omicron_common::api::internal::shared::ReifiedVpcRouteSet; -use omicron_common::api::internal::shared::ReifiedVpcRouteState; +use omicron_common::api::internal::shared::ResolvedVpcRoute; +use omicron_common::api::internal::shared::ResolvedVpcRouteSet; +use omicron_common::api::internal::shared::ResolvedVpcRouteState; use omicron_common::api::internal::shared::RouterId; use omicron_common::api::internal::shared::RouterTarget as ApiRouterTarget; use omicron_common::api::internal::shared::RouterVersion; @@ -55,10 +55,11 @@ use uuid::Uuid; // Prefix used to identify xde data links. const XDE_LINK_PREFIX: &str = "opte"; +/// Stored routes (and usage count) for a given VPC/subnet. #[derive(Debug, Clone)] struct RouteSet { version: Option, - routes: HashSet, + routes: HashSet, active_ports: usize, } @@ -66,17 +67,17 @@ struct RouteSet { struct PortManagerInner { log: Logger, - // Sequential identifier for each port on the system. + /// Sequential identifier for each port on the system. next_port_id: AtomicU64, - // IP address of the hosting sled on the underlay. + /// IP address of the hosting sled on the underlay. underlay_ip: Ipv6Addr, - // Map of all ports, keyed on the interface Uuid and its kind - // (which includes the Uuid of the parent instance or service) + /// Map of all ports, keyed on the interface Uuid and its kind + /// (which includes the Uuid of the parent instance or service) ports: Mutex>, - // Map of all current resolved routes. + /// Map of all current resolved routes. routes: Mutex>, } @@ -377,6 +378,10 @@ impl PortManager { (port, ticket) }; + // Check locally to see whether we have any routes from the + // control plane for this port already installed. If not, + // create a record to show that we're interested in receiving + // those routes. let mut routes = self.inner.routes.lock().unwrap(); let system_routes = routes.entry(port.system_router_key()).or_insert_with(|| { @@ -386,11 +391,11 @@ impl PortManager { // to reach out over the Internet *before* nexus is up to give // us real rules. The easiest bet is to instantiate these here. if is_service { - routes.insert(ReifiedVpcRoute { + routes.insert(ResolvedVpcRoute { dest: "0.0.0.0/0".parse().unwrap(), target: ApiRouterTarget::InternetGateway, }); - routes.insert(ReifiedVpcRoute { + routes.insert(ResolvedVpcRoute { dest: "::/0".parse().unwrap(), target: ApiRouterTarget::InternetGateway, }); @@ -399,7 +404,7 @@ impl PortManager { RouteSet { version: None, routes, active_ports: 0 } }); system_routes.active_ports += 1; - // Needed to get borrowck on our side, sadly. + // Clone is needed to get borrowck on our side, sadly. let system_routes = system_routes.clone(); let custom_routes = routes @@ -443,23 +448,23 @@ impl PortManager { Ok((port, ticket)) } - pub fn vpc_routes_list(&self) -> Vec { + pub fn vpc_routes_list(&self) -> Vec { let routes = self.inner.routes.lock().unwrap(); routes .iter() - .map(|(k, v)| ReifiedVpcRouteState { id: *k, version: v.version }) + .map(|(k, v)| ResolvedVpcRouteState { id: *k, version: v.version }) .collect() } pub fn vpc_routes_ensure( &self, - new_routes: Vec, + new_routes: Vec, ) -> Result<(), Error> { let mut routes = self.inner.routes.lock().unwrap(); let mut deltas = HashMap::new(); - for set in new_routes { + for new in new_routes { // Disregard any route information for a subnet we don't have. - let Some(old) = routes.get(&set.id) else { + let Some(old) = routes.get(&new.id) else { continue; }; @@ -468,34 +473,38 @@ impl PortManager { // If there's a UUID match, only update if vers increased, // otherwise take the update verbatim (including loss of version). let (to_add, to_delete): (HashSet<_>, HashSet<_>) = - match (old.version, set.version) { + match (old.version, new.version) { (Some(old_vers), Some(new_vers)) if !old_vers.is_replaced_by(&new_vers) => { continue; } _ => ( - set.routes.difference(&old.routes).cloned().collect(), - old.routes.difference(&set.routes).cloned().collect(), + new.routes.difference(&old.routes).cloned().collect(), + old.routes.difference(&new.routes).cloned().collect(), ), }; - deltas.insert(set.id, (to_add, to_delete)); + deltas.insert(new.id, (to_add, to_delete)); let active_ports = old.active_ports; routes.insert( - set.id, + new.id, RouteSet { - version: set.version, - routes: set.routes, + version: new.version, + routes: new.routes, active_ports, }, ); } + // Note: We're deliberately holding both locks here + // to prevent several nexuses computng and applying deltas + // out of order. let ports = self.inner.ports.lock().unwrap(); #[cfg(target_os = "illumos")] let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?; + // Propagate deltas out to all ports. for port in ports.values() { let system_id = port.system_router_key(); let system_delta = deltas.get(&system_id); diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index e66c89f86f..db783af78b 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(64, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(65, 0, 0); /// List of all past database schema versions, in *reverse* order /// diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index cd12cb6793..ce40e20501 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -441,58 +441,6 @@ impl DataStore { Ok(result) } - /// Lists all instances on in-service sleds with active Propolis VMM - /// processes, returning the instance along with the VMM on which it's - /// running, the sled on which the VMM is running, and the project that owns - /// the instance. - /// - /// The query performed by this function is paginated by the sled's UUID. - pub async fn instance_and_vpc_list_by_sled_agent( - &self, - opctx: &OpContext, - pagparams: &DataPageParams<'_, Uuid>, - ) -> ListResultVec<(Sled, Instance, Vmm, Project)> { - use crate::db::schema::{ - instance::dsl as instance_dsl, project::dsl as project_dsl, - sled::dsl as sled_dsl, vmm::dsl as vmm_dsl, - }; - opctx.authorize(authz::Action::Read, &authz::FLEET).await?; - let conn = self.pool_connection_authorized(opctx).await?; - - let result = paginated(sled_dsl::sled, sled_dsl::id, pagparams) - .filter(sled_dsl::time_deleted.is_null()) - .sled_filter(SledFilter::InService) - .inner_join( - vmm_dsl::vmm - .on(vmm_dsl::sled_id - .eq(sled_dsl::id) - .and(vmm_dsl::time_deleted.is_null())) - .inner_join( - instance_dsl::instance - .on(instance_dsl::id - .eq(vmm_dsl::instance_id) - .and(instance_dsl::time_deleted.is_null())) - .inner_join( - project_dsl::project.on(project_dsl::id - .eq(instance_dsl::project_id) - .and(project_dsl::time_deleted.is_null())), - ), - ), - ) - .sled_filter(SledFilter::InService) - .select(( - Sled::as_select(), - Instance::as_select(), - Vmm::as_select(), - Project::as_select(), - )) - .load_async::<(Sled, Instance, Vmm, Project)>(&*conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; - - Ok(result) - } - pub async fn project_delete_instance( &self, opctx: &OpContext, diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 6baa6f643f..c8e071684b 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -149,8 +149,8 @@ impl DataStore { // NIC of that instance. Accordingly, NIC create may cause dangling // entries to re-resolve to a valid instance (even if it is not yet // started). - // This will not trigger the RPW directly, we still need to do so - // in e.g. the instance watcher task. + // This will not trigger the route RPW directly, we still need to do + // so in e.g. the instance watcher task. if out.primary { self.vpc_increment_rpw_version(opctx, out.vpc_id) .await diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index fd90c8a7e6..acc6e278d3 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -66,7 +66,7 @@ use omicron_common::api::external::RouteTarget; use omicron_common::api::external::RouterRouteKind as ExternalRouteKind; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::Vni as ExternalVni; -use omicron_common::api::internal::shared::ReifiedVpcRoute; +use omicron_common::api::internal::shared::ResolvedVpcRoute; use omicron_common::api::internal::shared::RouterTarget; use ref_cast::RefCast; use std::collections::BTreeMap; @@ -1064,6 +1064,17 @@ impl DataStore { .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + // Unlink all subnets from this router. + // XXX: We might this want to error out before the delete fires. + use db::schema::vpc_subnet::dsl as vpc; + diesel::update(vpc::vpc_subnet) + .filter(vpc::time_deleted.is_null()) + .filter(vpc::custom_router_id.eq(authz_router.id())) + .set(vpc::custom_router_id.eq(Option::::None)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) } @@ -1312,7 +1323,8 @@ impl DataStore { // aren't something which they can meaningfully interact with, // so uuid stability on e.g. VPC rename is not a primary concern. // We make sure only to alter VPC subnet rules here: users may - // modify other system routes like internet gateways. + // modify other system routes like internet gateways (which are + // `RouteKind::Default`). let conn = self.pool_connection_authorized(opctx).await?; let log = opctx.log.clone(); self.transaction_retry_wrapper("vpc_subnet_route_reconcile") @@ -1364,7 +1376,7 @@ impl DataStore { } } - // Add/Remove routes. Retry if numebr is incorrect due to + // Add/Remove routes. Retry if number is incorrect due to // concurrent modification. let now = Utc::now(); let to_update = invalid.len(); @@ -1407,6 +1419,27 @@ impl DataStore { } } + // Verify that route set is exactly as intended, and rollback otherwise. + let current_rules: Vec = dsl::router_route + .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::vpc_router_id.eq(system_router_id)) + .select(RouterRoute::as_select()) + .load_async(&conn) + .await?; + + if current_rules.len() != expected_names.len() { + return Err(DieselError::RollbackTransaction) + } + + for rule in current_rules { + match (rule.kind.0, rule.target.0) { + (ExternalRouteKind::VpcSubnet, RouteTarget::Subnet(n)) + if expected_names.contains(Name::ref_cast(&n)) => {}, + _ => return Err(DieselError::RollbackTransaction), + } + } + Ok(()) }}).await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; @@ -1420,7 +1453,6 @@ impl DataStore { opctx: &OpContext, vpc_id: Uuid, ) -> LookupResult { - // use db::schema::vpc::dsl as vpc_dsl; use db::schema::vpc::dsl as vpc_dsl; use db::schema::vpc_router::dsl as router_dsl; @@ -1486,7 +1518,7 @@ impl DataStore { &self, opctx: &OpContext, vpc_router_id: Uuid, - ) -> Result, Error> { + ) -> Result, Error> { // Get all rules in target router. opctx.check_complex_operations_allowed()?; @@ -1511,7 +1543,12 @@ impl DataStore { all_rules.extend(batch); } - // XXX: transaction based on generation number? + // This is not in a transaction, because... + // We're not necessarily too concerned about getting partially + // updated state when resolving these names. See the header discussion + // in `nexus/src/app/background/vpc_routes.rs`: any state updates + // are followed by a version bump/notify, so we will be eventually + // consistent with route resolution. let mut subnet_names = HashSet::new(); let mut vpc_names = HashSet::new(); let mut inetgw_names = HashSet::new(); @@ -1544,7 +1581,7 @@ impl DataStore { } } - // TODO: transact these, and/or solve in fewer queries. + // TODO: This would be nice to solve in fewer queries. let mut subnets = HashMap::new(); for name in subnet_names.drain() { if let Ok((.., subnet)) = db::lookup::LookupPath::new(opctx, self) @@ -1576,7 +1613,7 @@ impl DataStore { .fetch() .await { - // XXX: currently an instance can have one primary, + // XXX: currently an instance can have one primary NIC, // and it is not dual-stack (v4 + v6). We need // to clarify what should be resolved in the v6 case. if let Ok(primary_nic) = self @@ -1676,11 +1713,11 @@ impl DataStore { }; if let (Some(dest), Some(target)) = (v4_dest, v4_target) { - out.insert(ReifiedVpcRoute { dest, target }); + out.insert(ResolvedVpcRoute { dest, target }); } if let (Some(dest), Some(target)) = (v6_dest, v6_target) { - out.insert(ReifiedVpcRoute { dest, target }); + out.insert(ResolvedVpcRoute { dest, target }); } } @@ -2507,4 +2544,20 @@ mod tests { routes } + + // Test to verify that VPC routers resolve to the primary addr + // of an instance NIC. + #[tokio::test] + async fn test_vpc_router_rule_instance_resolve() { + // use vpc_resolve_router_rules. + todo!() + } + + // Test to verify that VPC routers resolve rules intelligently + // across dual IPv4 / IPv6 targets/destinations. + #[tokio::test] + async fn test_vpc_router_rule_v4_v6_resolve() { + // use vpc_resolve_router_rules. + todo!() + } } diff --git a/nexus/db-queries/src/db/fixed_data/vpc.rs b/nexus/db-queries/src/db/fixed_data/vpc.rs index 6dffc11426..604e939680 100644 --- a/nexus/db-queries/src/db/fixed_data/vpc.rs +++ b/nexus/db-queries/src/db/fixed_data/vpc.rs @@ -23,7 +23,7 @@ pub static SERVICES_VPC_ROUTER_ID: Lazy = Lazy::new(|| { .expect("invalid uuid for builtin services vpc router id") }); -/// UUID of default route for built-in Services VPC. +/// UUID of default IPv4 route for built-in Services VPC. pub static SERVICES_VPC_DEFAULT_V4_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-074c-4000-8000-000000000002" @@ -31,6 +31,7 @@ pub static SERVICES_VPC_DEFAULT_V4_ROUTE_ID: Lazy = .expect("invalid uuid for builtin services vpc default route id") }); +/// UUID of default IPv6 route for built-in Services VPC. pub static SERVICES_VPC_DEFAULT_V6_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-074c-4000-8000-000000000003" diff --git a/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs b/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs index 45db9b7e0b..7b2cc468ab 100644 --- a/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs +++ b/nexus/db-queries/src/db/fixed_data/vpc_subnet.rs @@ -31,18 +31,21 @@ pub static NTP_VPC_SUBNET_ID: Lazy = Lazy::new(|| { .expect("invalid uuid for builtin boundary ntp vpc subnet id") }); +/// UUID of built-in subnet route VPC Subnet route for External DNS. pub static DNS_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-c470-4000-8000-000000000004" .parse() .expect("invalid uuid for builtin services vpc default route id") }); +/// UUID of built-in subnet route VPC Subnet route for Nexus. pub static NEXUS_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-c470-4000-8000-000000000005" .parse() .expect("invalid uuid for builtin services vpc default route id") }); +/// UUID of built-in subnet route VPC Subnet route for Boundary NTP. pub static NTP_VPC_SUBNET_ROUTE_ID: Lazy = Lazy::new(|| { "001de000-c470-4000-8000-000000000006" .parse() diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index 6cdf720ce2..359b86f939 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -15,7 +15,7 @@ use nexus_types::{ identity::Resource, }; use omicron_common::api::internal::shared::{ - ReifiedVpcRoute, ReifiedVpcRouteSet, RouterId, RouterVersion, + ResolvedVpcRoute, ResolvedVpcRouteSet, RouterId, RouterVersion, }; use serde_json::json; use std::collections::hash_map::Entry; @@ -78,7 +78,7 @@ impl BackgroundTask for VpcRouteManager { }) .collect(); - let mut known_rules: HashMap> = + let mut known_rules: HashMap> = HashMap::new(); let mut db_routers = HashMap::new(); let mut vni_to_vpc = HashMap::new(); @@ -184,7 +184,7 @@ impl BackgroundTask for VpcRouteManager { let mut to_push = Vec::new(); let mut set_rules = |id, version, routes| { - to_push.push(ReifiedVpcRouteSet { id, routes, version }); + to_push.push(ResolvedVpcRouteSet { id, routes, version }); }; // resolve into known_rules on an as-needed basis. @@ -199,7 +199,7 @@ impl BackgroundTask for VpcRouteManager { let router_id = db_router.id(); let version = RouterVersion { - generation: db_router.resolved_version as u64, + version: db_router.resolved_version as u64, router_id, }; diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index c84cf5ff20..22db026c4f 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -228,8 +228,6 @@ async fn svc_create_router_undo( Ok(()) } -// XX: possibly do these as a subsaga? - async fn svc_create_v4_route( sagactx: NexusActionContext, ) -> Result<(), ActionError> { diff --git a/nexus/src/app/vpc_router.rs b/nexus/src/app/vpc_router.rs index ae2fdffeeb..40b4c1de0f 100644 --- a/nexus/src/app/vpc_router.rs +++ b/nexus/src/app/vpc_router.rs @@ -118,8 +118,6 @@ impl super::Nexus { .await } - // TODO(now): When a router is deleted it should be unassociated w/ any subnets it may be associated with - // or trigger an error. pub(crate) async fn vpc_delete_router( &self, opctx: &OpContext, diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index d332a9de01..5480f0bcf1 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -967,10 +967,10 @@ "content": { "application/json": { "schema": { - "title": "Array_of_ReifiedVpcRouteState", + "title": "Array_of_ResolvedVpcRouteState", "type": "array", "items": { - "$ref": "#/components/schemas/ReifiedVpcRouteState" + "$ref": "#/components/schemas/ResolvedVpcRouteState" } } } @@ -991,10 +991,10 @@ "content": { "application/json": { "schema": { - "title": "Array_of_ReifiedVpcRouteSet", + "title": "Array_of_ResolvedVpcRouteSet", "type": "array", "items": { - "$ref": "#/components/schemas/ReifiedVpcRouteSet" + "$ref": "#/components/schemas/ResolvedVpcRouteSet" } } } @@ -4290,7 +4290,7 @@ "rack_subnet" ] }, - "ReifiedVpcRoute": { + "ResolvedVpcRoute": { "description": "A VPC route resolved into a concrete target.", "type": "object", "properties": { @@ -4306,8 +4306,8 @@ "target" ] }, - "ReifiedVpcRouteSet": { - "description": "An updated set of routes for a given", + "ResolvedVpcRouteSet": { + "description": "An updated set of routes for a given VPC and/or subnet.", "type": "object", "properties": { "id": { @@ -4316,7 +4316,7 @@ "routes": { "type": "array", "items": { - "$ref": "#/components/schemas/ReifiedVpcRoute" + "$ref": "#/components/schemas/ResolvedVpcRoute" }, "uniqueItems": true }, @@ -4334,8 +4334,8 @@ "routes" ] }, - "ReifiedVpcRouteState": { - "description": "Version information", + "ResolvedVpcRouteState": { + "description": "Version information for routes on a given VPC subnet.", "type": "object", "properties": { "id": { @@ -4384,7 +4384,7 @@ ] }, "RouterId": { - "description": "Implementation details on XXX", + "description": "Identifier for a VPC and/or subnet.", "type": "object", "properties": { "subnet": { @@ -4474,22 +4474,22 @@ ] }, "RouterVersion": { - "description": "XXX", + "description": "Information on the current parent router (and version) of a route set according to the control plane.", "type": "object", "properties": { - "generation": { - "type": "integer", - "format": "uint64", - "minimum": 0 - }, "router_id": { "type": "string", "format": "uuid" + }, + "version": { + "type": "integer", + "format": "uint64", + "minimum": 0 } }, "required": [ - "generation", - "router_id" + "router_id", + "version" ] }, "SemverVersion": { diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index fc70d10f3d..f354ea2fec 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3935,7 +3935,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '64.0.0', NULL) + (TRUE, NOW(), NOW(), '65.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index ff5ab393ea..3ec905960c 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -32,7 +32,7 @@ use omicron_common::api::internal::nexus::{ DiskRuntimeState, SledInstanceState, UpdateArtifactId, }; use omicron_common::api::internal::shared::{ - ReifiedVpcRouteSet, ReifiedVpcRouteState, SwitchPorts, + ResolvedVpcRouteSet, ResolvedVpcRouteState, SwitchPorts, }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -1036,7 +1036,7 @@ async fn bootstore_status( }] async fn list_vpc_routes( request_context: RequestContext, -) -> Result>, HttpError> { +) -> Result>, HttpError> { let sa = request_context.context(); Ok(HttpResponseOk(sa.list_vpc_routes())) } @@ -1048,7 +1048,7 @@ async fn list_vpc_routes( }] async fn set_vpc_routes( request_context: RequestContext, - body: TypedBody>, + body: TypedBody>, ) -> Result { let sa = request_context.context(); sa.set_vpc_routes(body.into_inner())?; diff --git a/sled-agent/src/probe_manager.rs b/sled-agent/src/probe_manager.rs index eabf3850af..40af604645 100644 --- a/sled-agent/src/probe_manager.rs +++ b/sled-agent/src/probe_manager.rs @@ -185,7 +185,7 @@ impl ProbeManagerInner { self.remove(current.difference(&target)).await; self.check(current.intersection(&target)).await; - // If we have created some new probes, we may (in future) need the control plane + // If we have created some new probes, we may need the control plane // to provide us with valid routes for the VPC the probe belongs to. if n_added > 0 { if let Err(e) = self diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index ee33733718..facbd9db88 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -49,7 +49,8 @@ use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; use omicron_common::api::internal::shared::{ - HostPortConfig, RackNetworkConfig, ReifiedVpcRouteSet, ReifiedVpcRouteState, + HostPortConfig, RackNetworkConfig, ResolvedVpcRouteSet, + ResolvedVpcRouteState, }; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, @@ -1095,13 +1096,13 @@ impl SledAgent { self.inner.bootstore.clone() } - pub fn list_vpc_routes(&self) -> Vec { + pub fn list_vpc_routes(&self) -> Vec { self.inner.port_manager.vpc_routes_list() } pub fn set_vpc_routes( &self, - routes: Vec, + routes: Vec, ) -> Result<(), Error> { self.inner.port_manager.vpc_routes_ensure(routes).map_err(Error::from) } From f217bd19a3a55f28b25f59c24a4a1c93de662aae Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 14:39:17 +0100 Subject: [PATCH 26/59] Self-review pt.2. --- nexus/src/app/background/vpc_routes.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index 359b86f939..f9b37d2175 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -35,10 +35,20 @@ impl VpcRouteManager { } } -// There's a sort of eventual consistency happening here. -// ... DETAIL XX ... -// version bumps must happen AFTER other changes occur in -// children etc. to keep this sane and working. :) +// This RPW doesn't concern itself overly much with resolved router targets +// and destinations being partial wrt. the current generation, in the same +// vein as how firewall rules are handled. Gating *pushing* this update on a +// generation number can be a bit more risky, but there's a sort of eventual +// consistency happening here that keeps this safe. +// +// Any location which updates name-resolvable state follows the pattern: +// * Update state. +// * Update (VPC-wide) router generation numbers. +// * Awaken this task. This might happen indirectly via e.g. instance start. +// +// As a result, any update which accidentally sees partial state will be followed +// by re-triggering this RPW with a higher generation number, giving us a re-resolved +// route set and pushing to any relevant sleds. impl BackgroundTask for VpcRouteManager { fn activate<'a>( &'a mut self, @@ -47,7 +57,6 @@ impl BackgroundTask for VpcRouteManager { async { let log = &opctx.log; - // XX: copied from omicron#5566 let sleds = match self .datastore .sled_list_all_batched(opctx, SledFilter::InService) @@ -99,7 +108,7 @@ impl BackgroundTask for VpcRouteManager { // based on the set of VNIs reported by this sled. // These provide the versions we'll stick with -- in the worst // case we push newer state to a sled with an older generation - // number, which + // number, which will be fixed up on the next activation. for set in &route_sets { let db_vni = Vni(set.id.vni); let maybe_vpc = vni_to_vpc.entry(set.id.vni); From b549044dade886b4218abf45a04394ee4a6ec5b9 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 23 May 2024 22:42:37 +0100 Subject: [PATCH 27/59] Test route resolution. --- nexus/db-queries/src/db/datastore/silo.rs | 2 +- nexus/db-queries/src/db/datastore/vpc.rs | 325 +++++++++++++++++----- nexus/src/app/background/vpc_routes.rs | 11 +- 3 files changed, 262 insertions(+), 76 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs index 0fd858b900..76efcf99b0 100644 --- a/nexus/db-queries/src/db/datastore/silo.rs +++ b/nexus/db-queries/src/db/datastore/silo.rs @@ -106,7 +106,7 @@ impl DataStore { Ok(()) } - async fn silo_create_query( + pub(crate) async fn silo_create_query( opctx: &OpContext, silo: Silo, ) -> Result, Error> { diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index acc6e278d3..a5ceb9a2c0 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -66,7 +66,6 @@ use omicron_common::api::external::RouteTarget; use omicron_common::api::external::RouterRouteKind as ExternalRouteKind; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::Vni as ExternalVni; -use omicron_common::api::internal::shared::ResolvedVpcRoute; use omicron_common::api::internal::shared::RouterTarget; use ref_cast::RefCast; use std::collections::BTreeMap; @@ -1518,16 +1517,16 @@ impl DataStore { &self, opctx: &OpContext, vpc_router_id: Uuid, - ) -> Result, Error> { + ) -> Result, Error> { // Get all rules in target router. opctx.check_complex_operations_allowed()?; let (.., authz_project, authz_vpc, authz_router) = db::lookup::LookupPath::new(opctx, self) .vpc_router_id(vpc_router_id) - .lookup_for(authz::Action::ListChildren) + .lookup_for(authz::Action::Read) .await - .internal_context("lookup built-in services project")?; + .internal_context("lookup router by id for rules")?; let mut paginator = Paginator::new(SQL_BATCH_SIZE); let mut all_rules = vec![]; while let Some(p) = paginator.next() { @@ -1633,7 +1632,7 @@ impl DataStore { // how we should resolve name misses in route resolution. // This method adopts the same strategy: a lookup failure corresponds // to a NO-OP rule. - let mut out = HashSet::new(); + let mut out = HashMap::new(); for rule in all_rules { // Some dests/targets (e.g., subnet) resolve to *several* specifiers // to handle both v4 and v6. The user-facing API will prevent severe @@ -1712,12 +1711,18 @@ impl DataStore { RouteTarget::Vpc(_) => (None, None), }; + // XXX: Is there another way we should be handling destination + // collisions within a router? 'first/last wins' is fairly + // arbitrary when lookups are sorted on UUID, but it's + // unpredictable. + // It would be really useful to raise collisions and + // misses to users, somehow. if let (Some(dest), Some(target)) = (v4_dest, v4_target) { - out.insert(ResolvedVpcRoute { dest, target }); + out.insert(dest, target); } if let (Some(dest), Some(target)) = (v6_dest, v6_target) { - out.insert(ResolvedVpcRoute { dest, target }); + out.insert(dest, target); } } @@ -1775,6 +1780,7 @@ mod tests { use crate::db::datastore::test::sled_system_hardware_for_test; use crate::db::datastore::test_utils::datastore_test; use crate::db::datastore::test_utils::IneligibleSleds; + use crate::db::fixed_data::silo::DEFAULT_SILO; use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; use crate::db::model::Project; use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; @@ -2295,17 +2301,11 @@ mod tests { logctx.cleanup_successful(); } - // Test to verify that subnet CRUD operations are correctly - // reflected in the nexus-managed system router attached to a VPC. - #[tokio::test] - async fn test_vpc_system_router_sync_to_subnets() { - usdt::register_probes().unwrap(); - let logctx = - dev::test_setup_log("test_vpc_system_router_sync_to_subnets"); - let log = &logctx.log; - let mut db = test_setup_database(&logctx.log).await; - let (opctx, datastore) = datastore_test(&logctx, &db).await; - + async fn create_initial_vpc( + log: &slog::Logger, + opctx: &OpContext, + datastore: &DataStore, + ) -> (authz::Project, authz::Vpc, Vpc, authz::VpcRouter, VpcRouter) { // Create a project and VPC. let project_params = params::ProjectCreate { identity: IdentityMetadataCreateParams { @@ -2313,7 +2313,7 @@ mod tests { description: String::from("test project"), }, }; - let project = Project::new(Uuid::new_v4(), project_params); + let project = Project::new(DEFAULT_SILO.id(), project_params); let (authz_project, _) = datastore .project_create(&opctx, project) .await @@ -2368,28 +2368,30 @@ mod tests { }, ); - let (_, db_router) = datastore + let (authz_router, db_router) = datastore .vpc_create_router(&opctx, &authz_vpc, router) .await .unwrap(); - // InternetGateway route creation is handled by the saga proper, - // so we'll only have subnet routes here. Initially, we start with none: - verify_all_subnet_routes_in_router( - &opctx, - &datastore, - db_router.id(), - &[], - ) - .await; + (authz_project, authz_vpc, db_vpc, authz_router, db_router) + } - // Add a new subnet and we should get a new route. + async fn new_subnet_ez( + opctx: &OpContext, + datastore: &DataStore, + db_vpc: &Vpc, + authz_vpc: &authz::Vpc, + name: &str, + ip: [u8; 4], + prefix_len: u8, + ) -> (authz::VpcSubnet, VpcSubnet) { let ipv6_block = db_vpc .ipv6_prefix .random_subnet(external::Ipv6Net::VPC_SUBNET_IPV6_PREFIX_LENGTH) .map(|block| block.0) .unwrap(); - let (authz_sub0, sub0) = datastore + + datastore .vpc_create_subnet( &opctx, &authz_vpc, @@ -2397,13 +2399,13 @@ mod tests { Uuid::new_v4(), db_vpc.id(), IdentityMetadataCreateParams { - name: "s0".parse().unwrap(), - description: "The default subnet...".into(), + name: name.parse().unwrap(), + description: "A subnet...".into(), }, external::Ipv4Net( Ipv4Network::new( - core::net::Ipv4Addr::new(172, 30, 0, 0), - 22, + core::net::Ipv4Addr::from(ip), + prefix_len, ) .unwrap(), ), @@ -2411,7 +2413,45 @@ mod tests { ), ) .await - .unwrap(); + .unwrap() + } + + // Test to verify that subnet CRUD operations are correctly + // reflected in the nexus-managed system router attached to a VPC, + // and that these resolve to the v4/6 subnets of each. + #[tokio::test] + async fn test_vpc_system_router_sync_to_subnets() { + usdt::register_probes().unwrap(); + let logctx = + dev::test_setup_log("test_vpc_system_router_sync_to_subnets"); + let log = &logctx.log; + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + let (_, authz_vpc, db_vpc, _, db_router) = + create_initial_vpc(log, &opctx, &datastore).await; + + // InternetGateway route creation is handled by the saga proper, + // so we'll only have subnet routes here. Initially, we start with none: + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[], + ) + .await; + + // Add a new subnet and we should get a new route. + let (authz_sub0, sub0) = new_subnet_ez( + &opctx, + &datastore, + &db_vpc, + &authz_vpc, + "s0", + [172, 30, 0, 0], + 22, + ) + .await; verify_all_subnet_routes_in_router( &opctx, @@ -2422,34 +2462,16 @@ mod tests { .await; // Add another, and get another route. - let ipv6_block = db_vpc - .ipv6_prefix - .random_subnet(external::Ipv6Net::VPC_SUBNET_IPV6_PREFIX_LENGTH) - .map(|block| block.0) - .unwrap(); - let (_, sub1) = datastore - .vpc_create_subnet( - &opctx, - &authz_vpc, - db::model::VpcSubnet::new( - Uuid::new_v4(), - db_vpc.id(), - IdentityMetadataCreateParams { - name: "s1".parse().unwrap(), - description: "A second subnet...".into(), - }, - external::Ipv4Net( - Ipv4Network::new( - core::net::Ipv4Addr::new(172, 31, 0, 0), - 22, - ) - .unwrap(), - ), - ipv6_block, - ), - ) - .await - .unwrap(); + let (_, sub1) = new_subnet_ez( + &opctx, + &datastore, + &db_vpc, + &authz_vpc, + "s1", + [172, 31, 0, 0], + 22, + ) + .await; verify_all_subnet_routes_in_router( &opctx, @@ -2542,6 +2564,35 @@ mod tests { assert_eq!(count, 1, "subnet {name} should appear exactly once") } + // Resolve the routes: we should have two for each entry: + let resolved = datastore + .vpc_resolve_router_rules(&opctx, router_id) + .await + .unwrap(); + assert_eq!(resolved.len(), 2 * subnets.len()); + + // And each subnet generates a v4->v4 and v6->v6. + for subnet in subnets { + assert!(resolved.iter().any(|(k, v)| { + *k == subnet.ipv4_block.0.into() + && match v { + RouterTarget::VpcSubnet(ip) => { + *ip == subnet.ipv4_block.0.into() + } + _ => false, + } + })); + assert!(resolved.iter().any(|(k, v)| { + *k == subnet.ipv6_block.0.into() + && match v { + RouterTarget::VpcSubnet(ip) => { + *ip == subnet.ipv6_block.0.into() + } + _ => false, + } + })); + } + routes } @@ -2549,15 +2600,143 @@ mod tests { // of an instance NIC. #[tokio::test] async fn test_vpc_router_rule_instance_resolve() { - // use vpc_resolve_router_rules. - todo!() - } + usdt::register_probes().unwrap(); + let logctx = + dev::test_setup_log("test_vpc_router_rule_instance_resolve"); + let log = &logctx.log; + let db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; - // Test to verify that VPC routers resolve rules intelligently - // across dual IPv4 / IPv6 targets/destinations. - #[tokio::test] - async fn test_vpc_router_rule_v4_v6_resolve() { - // use vpc_resolve_router_rules. - todo!() + let (authz_project, authz_vpc, db_vpc, authz_router, _) = + create_initial_vpc(log, &opctx, &datastore).await; + + // Create a subnet for an instance to live in. + let (authz_sub0, sub0) = new_subnet_ez( + &opctx, + &datastore, + &db_vpc, + &authz_vpc, + "s0", + [172, 30, 0, 0], + 22, + ) + .await; + + // Add a rule pointing to the instance before it is created. + // We're commiting some minor data integrity sins by putting + // these into a system router, but that's irrelevant to resolution. + let inst_name = "insty".parse::().unwrap(); + let _ = datastore + .router_create_route( + &opctx, + &authz_router, + RouterRoute::new( + Uuid::new_v4(), + authz_router.id(), + external::RouterRouteKind::Custom, + params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "to-vpn".parse().unwrap(), + description: "A rule...".into(), + }, + target: external::RouteTarget::Instance( + inst_name.clone(), + ), + destination: external::RouteDestination::IpNet( + "192.168.0.0/16".parse().unwrap(), + ), + }, + ), + ) + .await + .unwrap(); + + // Resolve the rules: we will have two entries generated by the + // VPC subnet (v4, v6). + let routes = datastore + .vpc_resolve_router_rules(&opctx, authz_router.id()) + .await + .unwrap(); + + assert_eq!(routes.len(), 2); + + // Create an instance, this will have no effect for now as + // the instance lacks a NIC. + let db_inst = datastore + .project_create_instance( + &opctx, + &authz_project, + db::model::Instance::new( + Uuid::new_v4(), + authz_project.id(), + ¶ms::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: inst_name.clone(), + description: "An instance...".into(), + }, + ncpus: external::InstanceCpuCount(1), + memory: 10.into(), + hostname: "insty".parse().unwrap(), + user_data: vec![], + network_interfaces: + params::InstanceNetworkInterfaceAttachment::None, + external_ips: vec![], + disks: vec![], + ssh_public_keys: None, + start: false, + }, + ), + ) + .await + .unwrap(); + let (.., authz_instance) = + db::lookup::LookupPath::new(&opctx, &datastore) + .instance_id(db_inst.id()) + .lookup_for(authz::Action::CreateChild) + .await + .unwrap(); + + let routes = datastore + .vpc_resolve_router_rules(&opctx, authz_router.id()) + .await + .unwrap(); + + assert_eq!(routes.len(), 2); + + // Create a primary NIC on the instance; the route can now resolve + // to the instance's IP. + let nic = datastore + .instance_create_network_interface( + &opctx, + &authz_sub0, + &authz_instance, + IncompleteNetworkInterface::new_instance( + Uuid::new_v4(), + db_inst.id(), + sub0, + IdentityMetadataCreateParams { + name: "nic".parse().unwrap(), + description: "A NIC...".into(), + }, + None, + ) + .unwrap(), + ) + .await + .unwrap(); + + let routes = datastore + .vpc_resolve_router_rules(&opctx, authz_router.id()) + .await + .unwrap(); + + // Verify we now have a route pointing at this instance. + assert_eq!(routes.len(), 3); + assert!(routes.iter().any(|(k, v)| (*k + == "192.168.0.0/16".parse::().unwrap()) + && match v { + RouterTarget::Ip(ip) => *ip == nic.ip.ip(), + _ => false, + })); } } diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index f9b37d2175..f305990a22 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -238,8 +238,15 @@ impl BackgroundTask for VpcRouteManager { .await { Ok(rules) => { - set_rules(set.id, Some(version), rules.clone()); - known_rules.insert(router_id, rules); + let collapsed: HashSet<_> = rules + .into_iter() + .map(|(dest, target)| ResolvedVpcRoute { + dest, + target, + }) + .collect(); + set_rules(set.id, Some(version), collapsed.clone()); + known_rules.insert(router_id, collapsed); } Err(e) => { error!( From f02535e1e7fe82d6c93d19c64ead0c7e522d3bc8 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 24 May 2024 11:23:08 +0100 Subject: [PATCH 28/59] Accidentally ended up on the wrong maghemite. --- Cargo.lock | 8 ++++---- Cargo.toml | 4 ++-- workspace-hack/Cargo.toml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 728722907e..c9401f02cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" dependencies = [ "backtrace", ] @@ -1591,7 +1591,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=a557af774b2c3cfe1e9e4e27de9ad85de6a02a98#a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source = "git+https://github.com/oxidecomputer/maghemite?rev=c9824727eedc66d4920e42e7260df05050841ab8#c9824727eedc66d4920e42e7260df05050841ab8" dependencies = [ "percent-encoding", "progenitor", @@ -4295,7 +4295,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=a557af774b2c3cfe1e9e4e27de9ad85de6a02a98#a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" +source = "git+https://github.com/oxidecomputer/maghemite?rev=c9824727eedc66d4920e42e7260df05050841ab8#c9824727eedc66d4920e42e7260df05050841ab8" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index b70fbe25c9..ce52c84307 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -312,8 +312,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.12" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "c9824727eedc66d4920e42e7260df05050841ab8" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "c9824727eedc66d4920e42e7260df05050841ab8" } multimap = "0.10.0" nexus-client = { path = "clients/nexus-client" } nexus-config = { path = "nexus-config" } diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index d8c9e7c634..0ed8fbe17a 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -19,7 +19,7 @@ workspace = true [dependencies] ahash = { version = "0.8.11" } aho-corasick = { version = "1.1.3" } -anyhow = { version = "1.0.83", features = ["backtrace"] } +anyhow = { version = "1.0.86", features = ["backtrace"] } base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } @@ -124,7 +124,7 @@ zeroize = { version = "1.7.0", features = ["std", "zeroize_derive"] } [build-dependencies] ahash = { version = "0.8.11" } aho-corasick = { version = "1.1.3" } -anyhow = { version = "1.0.83", features = ["backtrace"] } +anyhow = { version = "1.0.86", features = ["backtrace"] } base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } From 880378ad372713ce780f6acd5945a6d58aae8d39 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 24 May 2024 13:46:14 +0100 Subject: [PATCH 29/59] Hook VPC checks into sim-sled-agent, instance networking tests. --- nexus/db-queries/src/db/datastore/silo.rs | 2 +- nexus/tests/integration_tests/instances.rs | 109 ++++++++++++++++++++- sled-agent/src/sim/http_entrypoints.rs | 30 +++++- sled-agent/src/sim/sled_agent.rs | 72 +++++++++++++- 4 files changed, 209 insertions(+), 4 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs index 76efcf99b0..0fd858b900 100644 --- a/nexus/db-queries/src/db/datastore/silo.rs +++ b/nexus/db-queries/src/db/datastore/silo.rs @@ -106,7 +106,7 @@ impl DataStore { Ok(()) } - pub(crate) async fn silo_create_query( + async fn silo_create_query( opctx: &OpContext, silo: Silo, ) -> Result, Error> { diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 51e2552e85..4b27c5038a 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -18,6 +18,7 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID; use nexus_db_queries::db::lookup::LookupPath; +use nexus_db_queries::db::DataStore; use nexus_test_interface::NexusServer; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; @@ -60,6 +61,8 @@ use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::Name; use omicron_common::api::external::NameOrId; use omicron_common::api::external::Vni; +use omicron_common::api::internal::shared::ResolvedVpcRoute; +use omicron_common::api::internal::shared::RouterId; use omicron_nexus::app::MAX_MEMORY_BYTES_PER_INSTANCE; use omicron_nexus::app::MAX_VCPU_PER_INSTANCE; use omicron_nexus::app::MIN_MEMORY_BYTES_PER_INSTANCE; @@ -68,6 +71,7 @@ use omicron_nexus::TestInterfaces as _; use omicron_sled_agent::sim::SledAgent; use omicron_test_utils::dev::poll::wait_for_condition; use sled_agent_client::TestInterfaces as _; +use std::collections::HashSet; use std::convert::TryFrom; use std::net::Ipv4Addr; use std::sync::Arc; @@ -670,6 +674,29 @@ async fn test_instance_start_creates_networking_state( for agent in &sled_agents { assert_sled_v2p_mappings(agent, &nics[0], guest_nics[0].vni).await; } + + // Ensure that the target sled agent for our instance has received + // up-to-date VPC routes. + let with_vmm = datastore + .instance_fetch_with_vmm(&opctx, &authz_instance) + .await + .unwrap(); + + let mut checked = false; + for agent in &sled_agents { + if Some(agent.id) == with_vmm.sled_id() { + assert_sled_vpc_routes( + agent, + &opctx, + datastore, + nics[0].subnet_id, + guest_nics[0].vni, + ) + .await; + checked = true; + } + } + assert!(checked); } #[nexus_test] @@ -769,7 +796,9 @@ async fn test_instance_migrate(cptestctx: &ControlPlaneTestContext) { } #[nexus_test] -async fn test_instance_migrate_v2p(cptestctx: &ControlPlaneTestContext) { +async fn test_instance_migrate_v2p_and_routes( + cptestctx: &ControlPlaneTestContext, +) { let client = &cptestctx.external_client; let apictx = &cptestctx.server.server_context(); let nexus = &apictx.nexus; @@ -895,6 +924,15 @@ async fn test_instance_migrate_v2p(cptestctx: &ControlPlaneTestContext) { if sled_agent.id != dst_sled_id { assert_sled_v2p_mappings(sled_agent, &nics[0], guest_nics[0].vni) .await; + } else { + assert_sled_vpc_routes( + sled_agent, + &opctx, + datastore, + nics[0].subnet_id, + guest_nics[0].vni, + ) + .await; } } } @@ -4679,6 +4717,75 @@ async fn assert_sled_v2p_mappings( .expect("matching v2p mapping should be present"); } +/// Asserts that supplied sled agent's most recent VPC route sets +/// contain up-to-date routes for a known subnet. +pub async fn assert_sled_vpc_routes( + sled_agent: &Arc, + opctx: &OpContext, + datastore: &DataStore, + subnet_id: Uuid, + vni: Vni, +) { + let (.., authz_vpc, _, db_subnet) = LookupPath::new(opctx, datastore) + .vpc_subnet_id(subnet_id) + .fetch() + .await + .unwrap(); + + let custom_routes: HashSet<_> = + if let Some(router_id) = db_subnet.custom_router_id { + datastore + .vpc_resolve_router_rules(opctx, router_id) + .await + .unwrap() + .into_iter() + .map(|(dest, target)| ResolvedVpcRoute { dest, target }) + .collect() + } else { + Default::default() + }; + + let (.., vpc) = LookupPath::new(opctx, datastore) + .vpc_id(authz_vpc.id()) + .fetch() + .await + .unwrap(); + + let system_routes: HashSet<_> = datastore + .vpc_resolve_router_rules(opctx, vpc.system_router_id) + .await + .unwrap() + .into_iter() + .map(|(dest, target)| ResolvedVpcRoute { dest, target }) + .collect(); + + assert!(!system_routes.is_empty()); + + let condition = || async { + let vpc_routes = sled_agent.vpc_routes.lock().await; + let sys_routes_found = vpc_routes.iter().any(|(id, set)| { + *id == RouterId { vni, subnet: None } && set.routes == system_routes + }); + let custom_routes_found = vpc_routes.iter().any(|(id, set)| { + *id == RouterId { vni, subnet: Some(db_subnet.ipv4_block.0.into()) } + && set.routes == custom_routes + }); + + if sys_routes_found && custom_routes_found { + Ok(()) + } else { + Err(CondCheckError::NotYet::<()>) + } + }; + wait_for_condition( + condition, + &Duration::from_secs(1), + &Duration::from_secs(30), + ) + .await + .expect("matching vpc routes should be present"); +} + /// Simulate completion of an ongoing instance state transition. To do this, we /// have to look up the instance, then get the sled agent associated with that /// instance, and then tell it to finish simulating whatever async transition is diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index ae1318a8b1..977835e3ca 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -24,7 +24,9 @@ use illumos_utils::opte::params::VirtualNetworkInterfaceHost; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::UpdateArtifactId; -use omicron_common::api::internal::shared::SwitchPorts; +use omicron_common::api::internal::shared::{ + ResolvedVpcRouteSet, ResolvedVpcRouteState, SwitchPorts, +}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_storage::resources::DisksManagementResult; @@ -63,6 +65,8 @@ pub fn api() -> SledApiDescription { api.register(omicron_zones_get)?; api.register(omicron_zones_put)?; api.register(sled_add)?; + api.register(list_vpc_routes)?; + api.register(set_vpc_routes)?; Ok(()) } @@ -507,3 +511,27 @@ async fn sled_add( ) -> Result { Ok(HttpResponseUpdatedNoContent()) } + +#[endpoint { + method = GET, + path = "/vpc-routes", +}] +async fn list_vpc_routes( + rqctx: RequestContext>, +) -> Result>, HttpError> { + let sa = rqctx.context(); + Ok(HttpResponseOk(sa.list_vpc_routes().await)) +} + +#[endpoint { + method = PUT, + path = "/vpc-routes", +}] +async fn set_vpc_routes( + rqctx: RequestContext>, + body: TypedBody>, +) -> Result { + let sa = rqctx.context(); + sa.set_vpc_routes(body.into_inner()).await; + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index d9308bf769..9463e5468f 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -37,7 +37,10 @@ use omicron_common::api::internal::nexus::{ use omicron_common::api::internal::nexus::{ InstanceRuntimeState, VmmRuntimeState, }; -use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::{ + RackNetworkConfig, ResolvedVpcRoute, ResolvedVpcRouteSet, + ResolvedVpcRouteState, RouterId, RouterVersion, +}; use omicron_common::disk::DiskIdentity; use omicron_uuid_kinds::ZpoolUuid; use propolis_client::{ @@ -77,6 +80,7 @@ pub struct SledAgent { Mutex>, PropolisClient)>>, /// lists of external IPs assigned to instances pub external_ips: Mutex>>, + pub vpc_routes: Mutex>, config: Config, fake_zones: Mutex, instance_ensure_state_error: Mutex>, @@ -189,6 +193,7 @@ impl SledAgent { disk_id_to_region_ids: Mutex::new(HashMap::new()), v2p_mappings: Mutex::new(HashSet::new()), external_ips: Mutex::new(HashMap::new()), + vpc_routes: Mutex::new(HashMap::new()), mock_propolis: Mutex::new(None), config: config.clone(), fake_zones: Mutex::new(OmicronZonesConfig { @@ -358,6 +363,26 @@ impl SledAgent { self.map_disk_ids_to_region_ids(&vcr).await?; } + let mut routes = self.vpc_routes.lock().await; + for nic in &hardware.nics { + let my_routers = [ + RouterId { + // system + vni: nic.vni, + subnet: None, + }, + RouterId { + // custom + vni: nic.vni, + subnet: Some(nic.subnet), + }, + ]; + + for router in my_routers { + routes.entry(router).or_default(); + } + } + Ok(instance_run_time_state) } @@ -861,4 +886,49 @@ impl SledAgent { ) { *self.fake_zones.lock().await = requested_zones; } + + pub async fn list_vpc_routes(&self) -> Vec { + let routes = self.vpc_routes.lock().await; + routes + .iter() + .map(|(k, v)| ResolvedVpcRouteState { id: *k, version: v.version }) + .collect() + } + + pub async fn set_vpc_routes(&self, new_routes: Vec) { + let mut routes = self.vpc_routes.lock().await; + for new in new_routes { + // Disregard any route information for a subnet we don't have. + let Some(old) = routes.get(&new.id) else { + continue; + }; + + // We have to handle subnet router changes, as well as + // spurious updates from multiple Nexus instances. + // If there's a UUID match, only update if vers increased, + // otherwise take the update verbatim (including loss of version). + match (old.version, new.version) { + (Some(old_vers), Some(new_vers)) + if !old_vers.is_replaced_by(&new_vers) => + { + continue; + } + _ => {} + }; + + routes.insert( + new.id, + RouteSet { version: new.version, routes: new.routes }, + ); + } + } +} + +/// Stored routes (and usage count) for a given VPC/subnet. +// NB: We aren't doing post count tracking here to unsubscribe +// from (VNI, subnet) pairs. +#[derive(Debug, Clone, Default)] +pub struct RouteSet { + pub version: Option, + pub routes: HashSet, } From 08c982e38c9f22da4b2622e8a380244ce2ecf020 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 24 May 2024 14:40:52 +0100 Subject: [PATCH 30/59] Correctly cleanup after new tests... --- nexus/db-queries/src/db/datastore/vpc.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index a5ceb9a2c0..ae4db4813f 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -2604,7 +2604,7 @@ mod tests { let logctx = dev::test_setup_log("test_vpc_router_rule_instance_resolve"); let log = &logctx.log; - let db = test_setup_database(&logctx.log).await; + let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; let (authz_project, authz_vpc, db_vpc, authz_router, _) = @@ -2738,5 +2738,8 @@ mod tests { RouterTarget::Ip(ip) => *ip == nic.ip.ip(), _ => false, })); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); } } From 821e241b11a8dc3ec53155664fd33b93e1e590d4 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 24 May 2024 21:23:24 +0100 Subject: [PATCH 31/59] Allow bind of subnet to custom router in create/update --- nexus/db-model/src/vpc_router.rs | 14 +- nexus/db-queries/src/db/datastore/vpc.rs | 77 + nexus/src/app/vpc_subnet.rs | 93 +- nexus/src/external_api/http_entrypoints.rs | 10 - nexus/tests/integration_tests/endpoints.rs | 2 + nexus/tests/integration_tests/instances.rs | 3 + .../integration_tests/subnet_allocation.rs | 1 + nexus/tests/integration_tests/vpc_subnets.rs | 5 + nexus/tests/output/nexus_tags.txt | 10 + nexus/types/src/external_api/params.rs | 12 + openapi/nexus.json | 1699 ++++++++++++++--- smf/sled-agent/non-gimlet/config-rss.toml | 21 +- smf/sled-agent/non-gimlet/config.toml | 2 +- 13 files changed, 1645 insertions(+), 304 deletions(-) diff --git a/nexus/db-model/src/vpc_router.rs b/nexus/db-model/src/vpc_router.rs index 51409c38d5..ce8fa97364 100644 --- a/nexus/db-model/src/vpc_router.rs +++ b/nexus/db-model/src/vpc_router.rs @@ -4,7 +4,8 @@ use super::{impl_enum_type, Generation, Name, RouterRoute}; use crate::collection::DatastoreCollectionConfig; -use crate::schema::{router_route, vpc_router}; +use crate::schema::{router_route, vpc_router, vpc_subnet}; +use crate::{DatastoreAttachTargetConfig, VpcSubnet}; use chrono::{DateTime, Utc}; use db_macros::Resource; use nexus_types::external_api::params; @@ -99,3 +100,14 @@ impl From for VpcRouterUpdate { } } } + +impl DatastoreAttachTargetConfig for VpcRouter { + type Id = Uuid; + + type CollectionIdColumn = vpc_router::dsl::id; + type CollectionTimeDeletedColumn = vpc_router::dsl::time_deleted; + + type ResourceIdColumn = vpc_subnet::dsl::id; + type ResourceCollectionIdColumn = vpc_subnet::dsl::custom_router_id; + type ResourceTimeDeletedColumn = vpc_subnet::dsl::time_deleted; +} diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index ae4db4813f..db8c6f12aa 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -9,6 +9,8 @@ use super::SQL_BATCH_SIZE; use crate::authz; use crate::context::OpContext; use crate::db; +use crate::db::collection_attach::AttachError; +use crate::db::collection_attach::DatastoreAttachTarget; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; @@ -935,6 +937,81 @@ impl DataStore { Ok(out) } + pub async fn vpc_subnet_set_custom_router( + &self, + opctx: &OpContext, + authz_subnet: &authz::VpcSubnet, + authz_router: &authz::VpcRouter, + ) -> Result { + opctx.authorize(authz::Action::Modify, authz_subnet).await?; + opctx.authorize(authz::Action::Read, authz_router).await?; + + use db::schema::vpc_router::dsl as router_dsl; + use db::schema::vpc_subnet::dsl as subnet_dsl; + + let query = VpcRouter::attach_resource( + authz_router.id(), + authz_subnet.id(), + router_dsl::vpc_router + .into_boxed() + .filter(router_dsl::kind.eq(VpcRouterKind::Custom)), + subnet_dsl::vpc_subnet.into_boxed(), + u32::MAX, + diesel::update(subnet_dsl::vpc_subnet).set(( + subnet_dsl::time_modified.eq(Utc::now()), + subnet_dsl::custom_router_id.eq(authz_router.id()), + )), + ); + + query + .attach_and_get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map(|(_, resource)| resource) + .map_err(|e| match e { + AttachError::CollectionNotFound => Error::not_found_by_id( + ResourceType::VpcRouter, + &authz_router.id(), + ), + AttachError::ResourceNotFound => Error::not_found_by_id( + ResourceType::VpcSubnet, + &authz_subnet.id(), + ), + // The only other failure reason can be an attempt to use a system router. + AttachError::NoUpdate { .. } => Error::invalid_request( + "cannot attach a system router to a VPC subnet", + ), + AttachError::DatabaseError(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + + pub async fn vpc_subnet_unset_custom_router( + &self, + opctx: &OpContext, + authz_subnet: &authz::VpcSubnet, + ) -> Result { + opctx.authorize(authz::Action::Modify, authz_subnet).await?; + + use db::schema::vpc_subnet::dsl; + + diesel::update(dsl::vpc_subnet) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_subnet.id())) + .set(dsl::custom_router_id.eq(Option::::None)) + .returning(VpcSubnet::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_subnet), + ) + }) + } + pub async fn subnet_list_instance_network_interfaces( &self, opctx: &OpContext, diff --git a/nexus/src/app/vpc_subnet.rs b/nexus/src/app/vpc_subnet.rs index 0e3affb470..05ce3ada87 100644 --- a/nexus/src/app/vpc_subnet.rs +++ b/nexus/src/app/vpc_subnet.rs @@ -107,7 +107,7 @@ impl super::Nexus { // See for // details. let subnet_id = Uuid::new_v4(); - let out = match params.ipv6_block { + let mut out = match params.ipv6_block { None => { const NUM_RETRIES: usize = 2; let mut retry = 0; @@ -213,6 +213,23 @@ impl super::Nexus { } }?; + // XX: rollback the creation if this fails? + if let Some(custom_router) = ¶ms.custom_router { + let (.., authz_subnet) = LookupPath::new(opctx, &self.db_datastore) + .vpc_subnet_id(out.id()) + .lookup_for(authz::Action::Modify) + .await?; + + out = self + .vpc_subnet_update_custom_router( + opctx, + &authz_vpc, + &authz_subnet, + Some(custom_router), + ) + .await?; + } + self.vpc_needed_notify_sleds(); Ok(out) @@ -235,8 +252,18 @@ impl super::Nexus { vpc_subnet_lookup: &lookup::VpcSubnet<'_>, params: ¶ms::VpcSubnetUpdate, ) -> UpdateResult { - let (.., authz_subnet) = + let (.., authz_vpc, authz_subnet) = vpc_subnet_lookup.lookup_for(authz::Action::Modify).await?; + + // Updating the subnet is a separate action. + self.vpc_subnet_update_custom_router( + opctx, + &authz_vpc, + &authz_subnet, + params.custom_router.as_ref(), + ) + .await?; + let out = self .db_datastore .vpc_update_subnet(&opctx, &authz_subnet, params.clone().into()) @@ -247,6 +274,68 @@ impl super::Nexus { Ok(out) } + async fn vpc_subnet_update_custom_router( + &self, + opctx: &OpContext, + authz_vpc: &authz::Vpc, + authz_subnet: &authz::VpcSubnet, + custom_router: Option<&NameOrId>, + ) -> UpdateResult { + // Resolve the VPC router, if specified. + let router_lookup = match custom_router { + Some(key @ NameOrId::Name(_)) => self + .vpc_router_lookup( + opctx, + params::RouterSelector { + project: None, + vpc: Some(NameOrId::Id(authz_vpc.id())), + router: key.clone(), + }, + ) + .map(Some), + Some(key @ NameOrId::Id(_)) => self + .vpc_router_lookup( + opctx, + params::RouterSelector { + project: None, + vpc: None, + router: key.clone(), + }, + ) + .map(Some), + None => Ok(None), + }?; + + let router_lookup = if let Some(l) = router_lookup { + let (.., rtr_authz_vpc, authz_router) = + l.lookup_for(authz::Action::Read).await?; + + if authz_vpc.id() != rtr_authz_vpc.id() { + return Err(Error::invalid_request( + "router and subnet must belong to the same VPC", + )); + } + + Some(authz_router) + } else { + None + }; + + if let Some(authz_router) = router_lookup { + self.db_datastore + .vpc_subnet_set_custom_router( + opctx, + &authz_subnet, + &authz_router, + ) + .await + } else { + self.db_datastore + .vpc_subnet_unset_custom_router(opctx, &authz_subnet) + .await + } + } + pub(crate) async fn vpc_delete_subnet( &self, opctx: &OpContext, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index e814df2b61..2678768b48 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -5446,7 +5446,6 @@ async fn vpc_firewall_rules_update( method = GET, path = "/v1/vpc-routers", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_list( rqctx: RequestContext, @@ -5486,7 +5485,6 @@ async fn vpc_router_list( method = GET, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_view( rqctx: RequestContext, @@ -5520,7 +5518,6 @@ async fn vpc_router_view( method = POST, path = "/v1/vpc-routers", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_create( rqctx: RequestContext, @@ -5556,7 +5553,6 @@ async fn vpc_router_create( method = DELETE, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_delete( rqctx: RequestContext, @@ -5590,7 +5586,6 @@ async fn vpc_router_delete( method = PUT, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_update( rqctx: RequestContext, @@ -5630,7 +5625,6 @@ async fn vpc_router_update( method = GET, path = "/v1/vpc-router-routes", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_list( rqctx: RequestContext, @@ -5672,7 +5666,6 @@ async fn vpc_router_route_list( method = GET, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_view( rqctx: RequestContext, @@ -5709,7 +5702,6 @@ async fn vpc_router_route_view( method = POST, path = "/v1/vpc-router-routes", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_create( rqctx: RequestContext, @@ -5745,7 +5737,6 @@ async fn vpc_router_route_create( method = DELETE, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_delete( rqctx: RequestContext, @@ -5781,7 +5772,6 @@ async fn vpc_router_route_delete( method = PUT, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], - unpublished = true, }] async fn vpc_router_route_update( rqctx: RequestContext, diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index cc73ab088c..a1c4fbb1cb 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -203,6 +203,7 @@ pub static DEMO_VPC_SUBNET_CREATE: Lazy = }, ipv4_block: Ipv4Net("10.1.2.3/8".parse().unwrap()), ipv6_block: None, + custom_router: None, }); // VPC Router used for testing @@ -1506,6 +1507,7 @@ pub static VERIFY_ENDPOINTS: Lazy> = Lazy::new(|| { name: None, description: Some("different".to_string()) }, + custom_router: None, }).unwrap() ), AllowedMethod::Delete, diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 4b27c5038a..41894392c1 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -1724,6 +1724,7 @@ async fn test_instance_with_new_custom_network_interfaces( }, ipv4_block: Ipv4Net("172.31.0.0/24".parse().unwrap()), ipv6_block: None, + custom_router: None, }; let _response = NexusRequest::objects_post( client, @@ -1870,6 +1871,7 @@ async fn test_instance_create_delete_network_interface( }, ipv4_block: Ipv4Net("172.31.0.0/24".parse().unwrap()), ipv6_block: None, + custom_router: None, }; let _response = NexusRequest::objects_post( client, @@ -2111,6 +2113,7 @@ async fn test_instance_update_network_interfaces( }, ipv4_block: Ipv4Net("172.31.0.0/24".parse().unwrap()), ipv6_block: None, + custom_router: None, }; let _response = NexusRequest::objects_post( client, diff --git a/nexus/tests/integration_tests/subnet_allocation.rs b/nexus/tests/integration_tests/subnet_allocation.rs index 0efc659890..68ccb15186 100644 --- a/nexus/tests/integration_tests/subnet_allocation.rs +++ b/nexus/tests/integration_tests/subnet_allocation.rs @@ -111,6 +111,7 @@ async fn test_subnet_allocation(cptestctx: &ControlPlaneTestContext) { // Use the minimum subnet size ipv4_block: Ipv4Net(subnet), ipv6_block: None, + custom_router: None, }; NexusRequest::objects_post(client, &subnets_url, &Some(&subnet_create)) .authn_as(AuthnMode::PrivilegedUser) diff --git a/nexus/tests/integration_tests/vpc_subnets.rs b/nexus/tests/integration_tests/vpc_subnets.rs index 0814512cf2..3ae77fd927 100644 --- a/nexus/tests/integration_tests/vpc_subnets.rs +++ b/nexus/tests/integration_tests/vpc_subnets.rs @@ -177,6 +177,7 @@ async fn test_vpc_subnets(cptestctx: &ControlPlaneTestContext) { }, ipv4_block, ipv6_block: Some(ipv6_block), + custom_router: None, }; let subnet: VpcSubnet = NexusRequest::objects_post(client, &subnets_url, &new_subnet) @@ -228,6 +229,7 @@ async fn test_vpc_subnets(cptestctx: &ControlPlaneTestContext) { }, ipv4_block, ipv6_block: Some(ipv6_block), + custom_router: None, }; let expected_error = format!( "IP address range '{}' conflicts with an existing subnet", @@ -255,6 +257,7 @@ async fn test_vpc_subnets(cptestctx: &ControlPlaneTestContext) { }, ipv4_block: other_ipv4_block, ipv6_block: other_ipv6_block, + custom_router: None, }; let error: dropshot::HttpErrorResponseBody = NexusRequest::new( RequestBuilder::new(client, Method::POST, &subnets_url) @@ -299,6 +302,7 @@ async fn test_vpc_subnets(cptestctx: &ControlPlaneTestContext) { }, ipv4_block, ipv6_block: None, + custom_router: None, }; let subnet2: VpcSubnet = NexusRequest::objects_post(client, &subnets_url, &new_subnet) @@ -327,6 +331,7 @@ async fn test_vpc_subnets(cptestctx: &ControlPlaneTestContext) { name: Some("new-name".parse().unwrap()), description: Some("another description".to_string()), }, + custom_router: None, }; NexusRequest::object_put(client, &subnet_url, Some(&update_params)) .authn_as(AuthnMode::PrivilegedUser) diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index a32fe5c4b9..35d8c32561 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -232,6 +232,16 @@ vpc_delete DELETE /v1/vpcs/{vpc} vpc_firewall_rules_update PUT /v1/vpc-firewall-rules vpc_firewall_rules_view GET /v1/vpc-firewall-rules vpc_list GET /v1/vpcs +vpc_router_create POST /v1/vpc-routers +vpc_router_delete DELETE /v1/vpc-routers/{router} +vpc_router_list GET /v1/vpc-routers +vpc_router_route_create POST /v1/vpc-router-routes +vpc_router_route_delete DELETE /v1/vpc-router-routes/{route} +vpc_router_route_list GET /v1/vpc-router-routes +vpc_router_route_update PUT /v1/vpc-router-routes/{route} +vpc_router_route_view GET /v1/vpc-router-routes/{route} +vpc_router_update PUT /v1/vpc-routers/{router} +vpc_router_view GET /v1/vpc-routers/{router} vpc_subnet_create POST /v1/vpc-subnets vpc_subnet_delete DELETE /v1/vpc-subnets/{subnet} vpc_subnet_list GET /v1/vpc-subnets diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 1b252c77cb..38826051fc 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -1219,6 +1219,13 @@ pub struct VpcSubnetCreate { /// be assigned if one is not provided. It must not overlap with any /// existing subnet in the VPC. pub ipv6_block: Option, + + /// An optional router, used to direct packets sent from hosts in this subnet + /// to any destination address. + /// + /// Custom routers apply in addition to the VPC-wide *system* router, and have + /// higher priority than + pub custom_router: Option, } /// Updateable properties of a `VpcSubnet` @@ -1226,6 +1233,9 @@ pub struct VpcSubnetCreate { pub struct VpcSubnetUpdate { #[serde(flatten)] pub identity: IdentityMetadataUpdateParams, + + /// XXX + pub custom_router: Option, } // VPC ROUTERS @@ -1251,7 +1261,9 @@ pub struct VpcRouterUpdate { pub struct RouterRouteCreate { #[serde(flatten)] pub identity: IdentityMetadataCreateParams, + /// The location that matched packets should be forwarded to. pub target: RouteTarget, + /// A CIDR block (or named subnet) which this route will apply to. pub destination: RouteDestination, } diff --git a/openapi/nexus.json b/openapi/nexus.json index 92af2a6b74..91d267c435 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -8346,13 +8346,14 @@ } } }, - "/v1/vpc-subnets": { + "/v1/vpc-router-routes": { "get": { "tags": [ "vpcs" ], - "summary": "List subnets", - "operationId": "vpc_subnet_list", + "summary": "List routes", + "description": "List the routes associated with a router in a particular VPC.", + "operationId": "vpc_router_route_list", "parameters": [ { "in": "query", @@ -8382,6 +8383,14 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "sort_by", @@ -8392,7 +8401,7 @@ { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8404,7 +8413,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetResultsPage" + "$ref": "#/components/schemas/RouterRouteResultsPage" } } } @@ -8418,7 +8427,7 @@ }, "x-dropshot-pagination": { "required": [ - "vpc" + "router" ] } }, @@ -8426,8 +8435,8 @@ "tags": [ "vpcs" ], - "summary": "Create subnet", - "operationId": "vpc_subnet_create", + "summary": "Create route", + "operationId": "vpc_router_route_create", "parameters": [ { "in": "query", @@ -8439,19 +8448,27 @@ }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetCreate" + "$ref": "#/components/schemas/RouterRouteCreate" } } }, @@ -8463,7 +8480,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8477,18 +8494,18 @@ } } }, - "/v1/vpc-subnets/{subnet}": { + "/v1/vpc-router-routes/{route}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch subnet", - "operationId": "vpc_subnet_view", + "summary": "Fetch route", + "operationId": "vpc_router_route_view", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8502,10 +8519,19 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8517,7 +8543,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8534,13 +8560,13 @@ "tags": [ "vpcs" ], - "summary": "Update subnet", - "operationId": "vpc_subnet_update", + "summary": "Update route", + "operationId": "vpc_router_route_update", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8554,10 +8580,18 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8567,7 +8601,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetUpdate" + "$ref": "#/components/schemas/RouterRouteUpdate" } } }, @@ -8579,7 +8613,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8596,13 +8630,13 @@ "tags": [ "vpcs" ], - "summary": "Delete subnet", - "operationId": "vpc_subnet_delete", + "summary": "Delete route", + "operationId": "vpc_router_route_delete", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8616,10 +8650,18 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8638,23 +8680,14 @@ } } }, - "/v1/vpc-subnets/{subnet}/network-interfaces": { + "/v1/vpc-routers": { "get": { "tags": [ "vpcs" ], - "summary": "List network interfaces", - "operationId": "vpc_subnet_list_network_interfaces", + "summary": "List routers", + "operationId": "vpc_router_list", "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "limit", @@ -8705,7 +8738,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" + "$ref": "#/components/schemas/VpcRouterResultsPage" } } } @@ -8718,89 +8751,30 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "vpc" + ] } - } - }, - "/v1/vpcs": { - "get": { + }, + "post": { "tags": [ "vpcs" ], - "summary": "List VPCs", - "operationId": "vpc_list", + "summary": "Create VPC router", + "operationId": "vpc_router_create", "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "project" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create VPC", - "operationId": "vpc_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8811,7 +8785,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcCreate" + "$ref": "#/components/schemas/VpcRouterCreate" } } }, @@ -8823,7 +8797,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -8837,18 +8811,18 @@ } } }, - "/v1/vpcs/{vpc}": { + "/v1/vpc-routers/{router}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch VPC", - "operationId": "vpc_view", + "summary": "Fetch router", + "operationId": "vpc_router_view", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8857,7 +8831,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8869,7 +8851,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -8886,13 +8868,13 @@ "tags": [ "vpcs" ], - "summary": "Update a VPC", - "operationId": "vpc_update", + "summary": "Update router", + "operationId": "vpc_router_update", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8901,7 +8883,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8911,7 +8901,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcUpdate" + "$ref": "#/components/schemas/VpcRouterUpdate" } } }, @@ -8923,7 +8913,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -8940,13 +8930,13 @@ "tags": [ "vpcs" ], - "summary": "Delete VPC", - "operationId": "vpc_delete", + "summary": "Delete router", + "operationId": "vpc_router_delete", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8955,7 +8945,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8973,43 +8971,671 @@ } } } - } - }, - "components": { - "schemas": { - "Address": { - "description": "An address tied to an address lot.", - "type": "object", - "properties": { - "address": { - "description": "The address and prefix length of this address.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] + }, + "/v1/vpc-subnets": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List subnets", + "operationId": "vpc_subnet_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } }, - "address_lot": { - "description": "The address lot this address is drawn from.", - "allOf": [ - { - "$ref": "#/components/schemas/NameOrId" - } - ] - } - }, - "required": [ - "address", - "address_lot" - ] - }, - "AddressConfig": { - "description": "A set of addresses associated with a port configuration.", - "type": "object", - "properties": { - "addresses": { - "description": "The set of addresses assigned to the port configuration.", - "type": "array", + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnetResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "vpc" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create subnet", + "operationId": "vpc_subnet_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnetCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpc-subnets/{subnet}": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "Fetch subnet", + "operationId": "vpc_subnet_view", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "vpcs" + ], + "summary": "Update subnet", + "operationId": "vpc_subnet_update", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnetUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcSubnet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "vpcs" + ], + "summary": "Delete subnet", + "operationId": "vpc_subnet_delete", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpc-subnets/{subnet}/network-interfaces": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List network interfaces", + "operationId": "vpc_subnet_list_network_interfaces", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/vpcs": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List VPCs", + "operationId": "vpc_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create VPC", + "operationId": "vpc_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpcs/{vpc}": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "Fetch VPC", + "operationId": "vpc_view", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "vpcs" + ], + "summary": "Update a VPC", + "operationId": "vpc_update", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "vpcs" + ], + "summary": "Delete VPC", + "operationId": "vpc_delete", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Address": { + "description": "An address tied to an address lot.", + "type": "object", + "properties": { + "address": { + "description": "The address and prefix length of this address.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "address_lot": { + "description": "The address lot this address is drawn from.", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + } + }, + "required": [ + "address", + "address_lot" + ] + }, + "AddressConfig": { + "description": "A set of addresses associated with a port configuration.", + "type": "object", + "properties": { + "addresses": { + "description": "The set of addresses assigned to the port configuration.", + "type": "array", "items": { "$ref": "#/components/schemas/Address" } @@ -15632,7 +16258,154 @@ "type": "string" }, "name": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "description", + "name" + ] + }, + "ProjectResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Project" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "ProjectRole": { + "type": "string", + "enum": [ + "admin", + "collaborator", + "viewer" + ] + }, + "ProjectRolePolicy": { + "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", + "type": "object", + "properties": { + "role_assignments": { + "description": "Roles directly assigned on this resource", + "type": "array", + "items": { + "$ref": "#/components/schemas/ProjectRoleRoleAssignment" + } + } + }, + "required": [ + "role_assignments" + ] + }, + "ProjectRoleRoleAssignment": { + "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", + "type": "object", + "properties": { + "identity_id": { + "type": "string", + "format": "uuid" + }, + "identity_type": { + "$ref": "#/components/schemas/IdentityType" + }, + "role_name": { + "$ref": "#/components/schemas/ProjectRole" + } + }, + "required": [ + "identity_id", + "identity_type", + "role_name" + ] + }, + "ProjectUpdate": { + "description": "Updateable properties of a `Project`", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, + "Rack": { + "description": "View of an Rack", + "type": "object", + "properties": { + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "id", + "time_created", + "time_modified" + ] + }, + "RackResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Rack" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "Role": { + "description": "View of a Role", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/RoleName" } }, "required": [ @@ -15640,7 +16413,14 @@ "name" ] }, - "ProjectResultsPage": { + "RoleName": { + "title": "A name for a built-in role", + "description": "Role names consist of two string components separated by dot (\".\").", + "type": "string", + "pattern": "[a-z-]+\\.[a-z-]+", + "maxLength": 63 + }, + "RoleResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -15648,7 +16428,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/Project" + "$ref": "#/components/schemas/Role" } }, "next_page": { @@ -15661,78 +16441,285 @@ "items" ] }, - "ProjectRole": { - "type": "string", - "enum": [ - "admin", - "collaborator", - "viewer" + "Route": { + "description": "A route to a destination network through a gateway address.", + "type": "object", + "properties": { + "dst": { + "description": "The route destination.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "gw": { + "description": "The route gateway.", + "type": "string", + "format": "ip" + }, + "vid": { + "nullable": true, + "description": "VLAN id the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "dst", + "gw" ] }, - "ProjectRolePolicy": { - "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", + "RouteConfig": { + "description": "Route configuration data associated with a switch port configuration.", "type": "object", "properties": { - "role_assignments": { - "description": "Roles directly assigned on this resource", + "routes": { + "description": "The set of routes assigned to a switch port.", "type": "array", "items": { - "$ref": "#/components/schemas/ProjectRoleRoleAssignment" + "$ref": "#/components/schemas/Route" } } }, "required": [ - "role_assignments" + "routes" ] }, - "ProjectRoleRoleAssignment": { - "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", - "type": "object", - "properties": { - "identity_id": { - "type": "string", - "format": "uuid" + "RouteDestination": { + "description": "A `RouteDestination` is used to match traffic with a routing rule, on the destination of that traffic.\n\nWhen traffic is to be sent to a destination that is within a given `RouteDestination`, the corresponding `RouterRoute` applies, and traffic will be forward to the `RouteTarget` for that rule.", + "oneOf": [ + { + "description": "Route applies to traffic destined for a specific IP address", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] }, - "identity_type": { - "$ref": "#/components/schemas/IdentityType" + { + "description": "Route applies to traffic destined for a specific IP subnet", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip_net" + ] + }, + "value": { + "$ref": "#/components/schemas/IpNet" + } + }, + "required": [ + "type", + "value" + ] }, - "role_name": { - "$ref": "#/components/schemas/ProjectRole" + { + "description": "Route applies to traffic destined for the given VPC.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Route applies to traffic", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] } - }, - "required": [ - "identity_id", - "identity_type", - "role_name" ] }, - "ProjectUpdate": { - "description": "Updateable properties of a `Project`", - "type": "object", - "properties": { - "description": { - "nullable": true, - "type": "string" + "RouteTarget": { + "description": "A `RouteTarget` describes the possible locations that traffic matching a route destination can be sent.", + "oneOf": [ + { + "description": "Forward traffic to a particular IP address.", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ip" + ] + }, + "value": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "type", + "value" + ] }, - "name": { - "nullable": true, - "allOf": [ - { + { + "description": "Forward traffic to a VPC", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to a VPC Subnet", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "subnet" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to a specific instance", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "instance" + ] + }, + "value": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Forward traffic to an internet gateway", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internet_gateway" + ] + }, + "value": { "$ref": "#/components/schemas/Name" } + }, + "required": [ + "type", + "value" + ] + }, + { + "description": "Drop matching traffic", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "drop" + ] + } + }, + "required": [ + "type" ] } - } + ] }, - "Rack": { - "description": "View of an Rack", + "RouterRoute": { + "description": "A route defines a rule that governs where traffic should be sent based on its destination.", "type": "object", "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "destination": { + "$ref": "#/components/schemas/RouteDestination" + }, "id": { "description": "unique, immutable, system-controlled identifier for each resource", "type": "string", "format": "uuid" }, + "kind": { + "description": "Describes the kind of router. Set at creation. `read-only`", + "allOf": [ + { + "$ref": "#/components/schemas/RouterRouteKind" + } + ] + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "target": { + "$ref": "#/components/schemas/RouteTarget" + }, "time_created": { "description": "timestamp when this resource was created", "type": "string", @@ -15742,59 +16729,93 @@ "description": "timestamp when this resource was last modified", "type": "string", "format": "date-time" + }, + "vpc_router_id": { + "description": "The ID of the VPC Router to which the route belongs", + "type": "string", + "format": "uuid" } }, "required": [ + "description", + "destination", "id", + "kind", + "name", + "target", "time_created", - "time_modified" - ] - }, - "RackResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/Rack" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" + "time_modified", + "vpc_router_id" ] }, - "Role": { - "description": "View of a Role", + "RouterRouteCreate": { + "description": "Create-time parameters for a `RouterRoute`", "type": "object", "properties": { "description": { "type": "string" }, + "destination": { + "description": "A CIDR block (or named subnet) which this route will apply to.", + "allOf": [ + { + "$ref": "#/components/schemas/RouteDestination" + } + ] + }, "name": { - "$ref": "#/components/schemas/RoleName" + "$ref": "#/components/schemas/Name" + }, + "target": { + "description": "The location that matched packets should be forwarded to.", + "allOf": [ + { + "$ref": "#/components/schemas/RouteTarget" + } + ] } }, "required": [ "description", - "name" + "destination", + "name", + "target" ] }, - "RoleName": { - "title": "A name for a built-in role", - "description": "Role names consist of two string components separated by dot (\".\").", - "type": "string", - "pattern": "[a-z-]+\\.[a-z-]+", - "maxLength": 63 + "RouterRouteKind": { + "description": "The kind of a `RouterRoute`\n\nThe kind determines certain attributes such as if the route is modifiable and describes how or where the route was created.", + "oneOf": [ + { + "description": "Determines the default destination of traffic, such as whether it goes to the internet or not.\n\n`Destination: An Internet Gateway` `Modifiable: true`", + "type": "string", + "enum": [ + "default" + ] + }, + { + "description": "Automatically added for each VPC Subnet in the VPC\n\n`Destination: A VPC Subnet` `Modifiable: false`", + "type": "string", + "enum": [ + "vpc_subnet" + ] + }, + { + "description": "Automatically added when VPC peering is established\n\n`Destination: A different VPC` `Modifiable: false`", + "type": "string", + "enum": [ + "vpc_peering" + ] + }, + { + "description": "Created by a user; see `RouteTarget`\n\n`Destination: User defined` `Modifiable: true`", + "type": "string", + "enum": [ + "custom" + ] + } + ] }, - "RoleResultsPage": { + "RouterRouteResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -15802,7 +16823,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/Role" + "$ref": "#/components/schemas/RouterRoute" } }, "next_page": { @@ -15815,50 +16836,32 @@ "items" ] }, - "Route": { - "description": "A route to a destination network through a gateway address.", + "RouterRouteUpdate": { + "description": "Updateable properties of a `RouterRoute`", "type": "object", "properties": { - "dst": { - "description": "The route destination.", + "description": { + "nullable": true, + "type": "string" + }, + "destination": { + "$ref": "#/components/schemas/RouteDestination" + }, + "name": { + "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/IpNet" + "$ref": "#/components/schemas/Name" } ] }, - "gw": { - "description": "The route gateway.", - "type": "string", - "format": "ip" - }, - "vid": { - "nullable": true, - "description": "VLAN id the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "dst", - "gw" - ] - }, - "RouteConfig": { - "description": "Route configuration data associated with a switch port configuration.", - "type": "object", - "properties": { - "routes": { - "description": "The set of routes assigned to a switch port.", - "type": "array", - "items": { - "$ref": "#/components/schemas/Route" - } + "target": { + "$ref": "#/components/schemas/RouteTarget" } }, "required": [ - "routes" + "destination", + "target" ] }, "SamlIdentityProvider": { @@ -18956,6 +19959,118 @@ "items" ] }, + "VpcRouter": { + "description": "A VPC router defines a series of rules that indicate where traffic should be sent depending on its destination.", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "kind": { + "$ref": "#/components/schemas/VpcRouterKind" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + }, + "vpc_id": { + "description": "The VPC to which the router belongs.", + "type": "string", + "format": "uuid" + } + }, + "required": [ + "description", + "id", + "kind", + "name", + "time_created", + "time_modified", + "vpc_id" + ] + }, + "VpcRouterCreate": { + "description": "Create-time parameters for a `VpcRouter`", + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "description", + "name" + ] + }, + "VpcRouterKind": { + "type": "string", + "enum": [ + "system", + "custom" + ] + }, + "VpcRouterResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/VpcRouter" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "VpcRouterUpdate": { + "description": "Updateable properties of a `VpcRouter`", + "type": "object", + "properties": { + "description": { + "nullable": true, + "type": "string" + }, + "name": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + } + } + }, "VpcSubnet": { "description": "A VPC subnet represents a logical grouping for instances that allows network traffic between them, within a IPv4 subnetwork or optionally an IPv6 subnetwork.", "type": "object", @@ -19030,6 +20145,15 @@ "description": "Create-time parameters for a `VpcSubnet`", "type": "object", "properties": { + "custom_router": { + "nullable": true, + "description": "An optional router, used to direct packets sent from hosts in this subnet to any destination address.\n\nCustom routers apply in addition to the VPC-wide *system* router, and have higher priority than", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + }, "description": { "type": "string" }, @@ -19085,6 +20209,15 @@ "description": "Updateable properties of a `VpcSubnet`", "type": "object", "properties": { + "custom_router": { + "nullable": true, + "description": "XXX", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + }, "description": { "nullable": true, "type": "string" diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index d897f7ba4b..595c53dc2f 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -33,7 +33,8 @@ external_dns_zone_name = "oxide.test" # the DNS domain delegated to the rack by the customer. Each of these addresses # must be contained in one of the "internal services" IP Pool ranges listed # below. -external_dns_ips = [ "192.168.1.20", "192.168.1.21" ] +# external_dns_ips = [ "192.168.1.20", "192.168.1.21" ] +external_dns_ips = [ "10.1.222.1", "10.1.222.2" ] # Initial TLS certificates for the external API # @@ -69,8 +70,10 @@ external_certificates = [] # # For more on this and what to put here, see docs/how-to-run.adoc. [[internal_services_ip_pool_ranges]] -first = "192.168.1.20" -last = "192.168.1.29" +# first = "192.168.1.20" +# last = "192.168.1.29" +first = "10.1.222.1" +last = "10.1.222.10" # TODO - this configuration is subject to change going forward. Ultimately these # parameters should be provided to the control plane via wicket, but we need to @@ -91,8 +94,10 @@ rack_subnet = "fd00:1122:3344:0100::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. -infra_ip_first = "192.168.1.30" -infra_ip_last = "192.168.1.30" +# infra_ip_first = "192.168.1.30" +# infra_ip_last = "192.168.1.30" +infra_ip_first = "10.1.222.10" +infra_ip_last = "10.1.222.10" # Configurations for BGP routers to run on the scrimlets. bgp = [] @@ -100,9 +105,11 @@ bgp = [] # You can configure multiple uplinks by repeating the following stanza [[rack_network_config.ports]] # Routes associated with this port. -routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] +# routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] +routes = [{nexthop = "10.0.0.1", destination = "0.0.0.0/0"}] # Addresses associated with this port. -addresses = ["192.168.1.30/24"] +# addresses = ["192.168.1.30/24"] +addresses = ["10.1.222.10/8"] # Name of the uplink port. This should always be "qsfp0" when using softnpu. port = "qsfp0" # The speed of this port. diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index 77ca52a647..e1b1b91b06 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -64,7 +64,7 @@ swap_device_size_gb = 64 # # If empty, this will be equivalent to the first result from: # $ dladm show-phys -p -o LINK -# data_link = "igb0" +data_link = "rge0" # On a multi-sled system, transit-mode Maghemite runs in the `oxz_switch` zone # to configure routes between sleds. This runs over the Sidecar's rear ports From 9c68888e8f845b6cc643cc866043aa47a8e337e5 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 28 May 2024 13:13:55 +0100 Subject: [PATCH 32/59] Fix VpcRouter field order, test usual attachment behaviour Turns out the only place that misordering cropped up is in the attach-colelction query, which was a bit nasty to unearth. --- nexus/db-model/src/vpc_router.rs | 2 +- nexus/db-queries/src/db/datastore/vpc.rs | 2 + nexus/test-utils/src/resource_helpers.rs | 29 +++ nexus/tests/integration_tests/vpc_routers.rs | 217 +++++++++++++++++++ smf/sled-agent/non-gimlet/config-rss.toml | 21 +- smf/sled-agent/non-gimlet/config.toml | 2 +- 6 files changed, 257 insertions(+), 16 deletions(-) diff --git a/nexus/db-model/src/vpc_router.rs b/nexus/db-model/src/vpc_router.rs index ce8fa97364..9ceb6e3945 100644 --- a/nexus/db-model/src/vpc_router.rs +++ b/nexus/db-model/src/vpc_router.rs @@ -42,8 +42,8 @@ pub struct VpcRouter { #[diesel(embed)] identity: VpcRouterIdentity, - pub vpc_id: Uuid, pub kind: VpcRouterKind, + pub vpc_id: Uuid, pub rcgen: Generation, pub resolved_version: i64, } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index db8c6f12aa..957c0cb65d 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1142,6 +1142,8 @@ impl DataStore { // Unlink all subnets from this router. // XXX: We might this want to error out before the delete fires. + // XXX: This will temporarily leave some hanging subnet attachments. We need to be sure + // these are safely handled (no unwrap, or remove via join here.) use db::schema::vpc_subnet::dsl as vpc; diesel::update(vpc::vpc_subnet) .filter(vpc::time_deleted.is_null()) diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 2aef32d37c..57e1113a91 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -26,6 +26,7 @@ use nexus_types::external_api::views::FloatingIp; use nexus_types::external_api::views::IpPool; use nexus_types::external_api::views::IpPoolRange; use nexus_types::external_api::views::User; +use nexus_types::external_api::views::VpcSubnet; use nexus_types::external_api::views::{Project, Silo, Vpc, VpcRouter}; use nexus_types::identity::Resource; use nexus_types::internal_api::params as internal_params; @@ -35,6 +36,8 @@ use omicron_common::api::external::Error; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::Instance; use omicron_common::api::external::InstanceCpuCount; +use omicron_common::api::external::Ipv4Net; +use omicron_common::api::external::Ipv6Net; use omicron_common::api::external::NameOrId; use omicron_common::disk::DiskIdentity; use omicron_sled_agent::sim::SledAgent; @@ -559,6 +562,32 @@ pub async fn create_vpc_with_error( .unwrap() } +pub async fn create_vpc_subnet( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + subnet_name: &str, + ipv4_block: Ipv4Net, + ipv6_block: Option, + custom_router: Option<&str>, +) -> VpcSubnet { + object_create( + &client, + &format!("/v1/vpc-subnets?project={project_name}&vpc={vpc_name}"), + ¶ms::VpcSubnetCreate { + identity: IdentityMetadataCreateParams { + name: subnet_name.parse().unwrap(), + description: "vpc description".to_string(), + }, + ipv4_block, + ipv6_block, + custom_router: custom_router + .map(|n| NameOrId::Name(n.parse().unwrap())), + }, + ) + .await +} + pub async fn create_router( client: &ClientTestContext, project_name: &str, diff --git a/nexus/tests/integration_tests/vpc_routers.rs b/nexus/tests/integration_tests/vpc_routers.rs index 0b931efbd7..d59dad4afd 100644 --- a/nexus/tests/integration_tests/vpc_routers.rs +++ b/nexus/tests/integration_tests/vpc_routers.rs @@ -9,14 +9,20 @@ use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::identity_eq; use nexus_test_utils::resource_helpers::create_router; +use nexus_test_utils::resource_helpers::create_vpc_subnet; +use nexus_test_utils::resource_helpers::object_delete; use nexus_test_utils::resource_helpers::objects_list_page_authz; use nexus_test_utils::resource_helpers::{create_project, create_vpc}; +use nexus_test_utils::resource_helpers::{object_put, object_put_error}; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; +use nexus_types::external_api::params::VpcSubnetUpdate; use nexus_types::external_api::views::VpcRouter; use nexus_types::external_api::views::VpcRouterKind; +use nexus_types::external_api::views::VpcSubnet; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::IdentityMetadataUpdateParams; +use omicron_common::api::external::Ipv4Net; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -253,6 +259,217 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { assert_eq!(router_same_name.vpc_id, vpc2.identity.id); } +#[nexus_test] +async fn test_vpc_routers_attach_to_subnet( + cptestctx: &ControlPlaneTestContext, +) { + // XXX: really clean this up. + let client = &cptestctx.external_client; + + // --- + // XX: copied from above + // + + // Create a project that we'll use for testing. + // This includes the vpc 'default'. + let project_name = "springfield-squidport"; + let _ = create_project(&client, project_name).await; + let vpc_name = "default"; + + let routers_url = + format!("/v1/vpc-routers?project={}&vpc={}", project_name, vpc_name); + let subnets_url = + format!("/v1/vpc-subnets?project={}&vpc={}", project_name, vpc_name); + + // get routers should have only the system router created w/ the VPC + let routers = + objects_list_page_authz::(client, &routers_url).await.items; + assert_eq!(routers.len(), 1); + assert_eq!(routers[0].kind, VpcRouterKind::System); + // + // XX: copied from above + // --- + + // Create a custom router for later use. + let router_name = "routy"; + let router = + create_router(&client, project_name, vpc_name, router_name).await; + assert_eq!(router.kind, VpcRouterKind::Custom); + + // Attaching a system router should fail. + let err = object_put_error( + client, + &format!( + "/v1/vpc-subnets/default?project={project_name}&vpc={vpc_name}" + ), + &VpcSubnetUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: None, + }, + custom_router: Some(routers[0].identity.id.into()), + }, + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!(err.message, "cannot attach a system router to a VPC subnet"); + + // Attaching a new custom router should succeed. + let default_subnet: VpcSubnet = object_put( + client, + &format!( + "/v1/vpc-subnets/default?project={project_name}&vpc={vpc_name}" + ), + &VpcSubnetUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: None, + }, + custom_router: Some(router.identity.id.into()), + }, + ) + .await; + assert_eq!(default_subnet.custom_router_id, Some(router.identity.id)); + + // Attaching a custom router to another subnet (same VPC) should succeed: + // ... at create time. + let subnet_name = "subnetty"; + let subnet2 = create_vpc_subnet( + &client, + &project_name, + &vpc_name, + &subnet_name, + Ipv4Net("192.168.0.0/24".parse().unwrap()), + None, + Some(router_name), + ) + .await; + assert_eq!(subnet2.custom_router_id, Some(router.identity.id)); + + // ... and via update. + let subnet_name = "subnettier"; + let _ = create_vpc_subnet( + &client, + &project_name, + &vpc_name, + &subnet_name, + Ipv4Net("192.168.1.0/24".parse().unwrap()), + None, + None, + ) + .await; + + let subnet3: VpcSubnet = object_put( + client, + &format!( + "/v1/vpc-subnets/{subnet_name}?project={project_name}&vpc={vpc_name}", + ), + &VpcSubnetUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: None, + }, + custom_router: Some(router.identity.id.into()), + }, + ) + .await; + + assert_eq!(subnet3.custom_router_id, Some(router.identity.id)); + + // Attaching a custom router to another VPC's subnet should fail. + create_vpc(&client, project_name, "vpc1").await; + let err = object_put_error( + client, + &format!("/v1/vpc-subnets/default?project={project_name}&vpc=vpc1"), + &VpcSubnetUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: None, + }, + custom_router: Some(router.identity.id.into()), + }, + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!(err.message, "router and subnet must belong to the same VPC"); + + // Deleting a custom router should detach from all these subnets. + object_delete( + &client, + &format!( + "/v1/vpc-routers/{router_name}?vpc={}&project={project_name}", + "default" + ), + ) + .await; + + for subnet in + objects_list_page_authz::(client, &subnets_url).await.items + { + assert!(subnet.custom_router_id.is_none()); + } +} + +#[nexus_test] +async fn test_vpc_routers_custom_route_at_instance( + cptestctx: &ControlPlaneTestContext, +) { + let _client = &cptestctx.external_client; + + // Attempting to delete a system router should fail. + + // Attempting to add a new route to a system router should fail. + + // Attempting to modify/delete a VPC subnet route should fail. + + // Modifying the target of a Default (gateway) route should succeed. + + todo!() +} + +#[nexus_test] +async fn test_vpc_routers_modify_system_routes( + cptestctx: &ControlPlaneTestContext, +) { + let _client = &cptestctx.external_client; + + // Attempting to delete a system router should fail. + + // Attempting to add a new route to a system router should fail. + + // Attempting to modify/delete a VPC subnet route should fail. + + // Modifying the target of a Default (gateway) route should succeed. + + todo!() +} + +#[nexus_test] +async fn test_vpc_routers_internet_gateway_target( + cptestctx: &ControlPlaneTestContext, +) { + let _client = &cptestctx.external_client; + + // Internet gateways are not fully supported: only 'inetgw:outbound' + // is a valid choice. + + // This can be used in both system and custom routers. + + todo!() +} + +#[nexus_test] +async fn test_vpc_routers_disallowed_custom_targets( + cptestctx: &ControlPlaneTestContext, +) { + let _client = &cptestctx.external_client; + + // Neither 'vpc:xxx' nor 'subnet:xxx' can be specified as route targets + // in custom routers. + + todo!() +} + fn routers_eq(sn1: &VpcRouter, sn2: &VpcRouter) { identity_eq(&sn1.identity, &sn2.identity); assert_eq!(sn1.vpc_id, sn2.vpc_id); diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index 595c53dc2f..d897f7ba4b 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -33,8 +33,7 @@ external_dns_zone_name = "oxide.test" # the DNS domain delegated to the rack by the customer. Each of these addresses # must be contained in one of the "internal services" IP Pool ranges listed # below. -# external_dns_ips = [ "192.168.1.20", "192.168.1.21" ] -external_dns_ips = [ "10.1.222.1", "10.1.222.2" ] +external_dns_ips = [ "192.168.1.20", "192.168.1.21" ] # Initial TLS certificates for the external API # @@ -70,10 +69,8 @@ external_certificates = [] # # For more on this and what to put here, see docs/how-to-run.adoc. [[internal_services_ip_pool_ranges]] -# first = "192.168.1.20" -# last = "192.168.1.29" -first = "10.1.222.1" -last = "10.1.222.10" +first = "192.168.1.20" +last = "192.168.1.29" # TODO - this configuration is subject to change going forward. Ultimately these # parameters should be provided to the control plane via wicket, but we need to @@ -94,10 +91,8 @@ rack_subnet = "fd00:1122:3344:0100::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. -# infra_ip_first = "192.168.1.30" -# infra_ip_last = "192.168.1.30" -infra_ip_first = "10.1.222.10" -infra_ip_last = "10.1.222.10" +infra_ip_first = "192.168.1.30" +infra_ip_last = "192.168.1.30" # Configurations for BGP routers to run on the scrimlets. bgp = [] @@ -105,11 +100,9 @@ bgp = [] # You can configure multiple uplinks by repeating the following stanza [[rack_network_config.ports]] # Routes associated with this port. -# routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] -routes = [{nexthop = "10.0.0.1", destination = "0.0.0.0/0"}] +routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] # Addresses associated with this port. -# addresses = ["192.168.1.30/24"] -addresses = ["10.1.222.10/8"] +addresses = ["192.168.1.30/24"] # Name of the uplink port. This should always be "qsfp0" when using softnpu. port = "qsfp0" # The speed of this port. diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index e1b1b91b06..77ca52a647 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -64,7 +64,7 @@ swap_device_size_gb = 64 # # If empty, this will be equivalent to the first result from: # $ dladm show-phys -p -o LINK -data_link = "rge0" +# data_link = "igb0" # On a multi-sled system, transit-mode Maghemite runs in the `oxz_switch` zone # to configure routes between sleds. This runs over the Sidecar's rear ports From e886d167b33a8991a2a17be43a4ed12fe586fd57 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 28 May 2024 17:43:45 +0100 Subject: [PATCH 33/59] Fix custom router listing. Very, very silly filter on the VPC ID in there... --- nexus/db-queries/src/db/datastore/vpc.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index ae4db4813f..ea3a7d5ca0 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1496,8 +1496,9 @@ impl DataStore { .eq(subnet_dsl::custom_router_id)), ) .filter(subnet_dsl::time_deleted.is_null()) - .filter(subnet_dsl::vpc_id.is_null()) + .filter(subnet_dsl::vpc_id.eq(vpc_id)) .filter(router_dsl::time_deleted.is_null()) + .filter(router_dsl::vpc_id.eq(vpc_id)) .select((VpcSubnet::as_select(), VpcRouter::as_select())) .load_async(&*self.pool_connection_authorized(opctx).await?) .await From e78c51f03399034b5e9054449c137c82bc3eab14 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 29 May 2024 12:08:36 +0100 Subject: [PATCH 34/59] First shot at spoof-prevention prevention. Named, for now, 'transit IPs'. Not sure what the best wayy to alter them is but we'll see what mileage this gets us. --- common/src/api/external/mod.rs | 4 +++ common/src/api/internal/shared.rs | 1 + illumos-utils/src/opte/port_manager.rs | 16 +++++++++++ nexus/db-model/src/network_interface.rs | 26 +++++++++++++++--- nexus/db-model/src/omicron_zone_config.rs | 1 + nexus/db-model/src/schema.rs | 2 ++ nexus/db-model/src/schema_versions.rs | 3 ++- .../src/db/datastore/network_interface.rs | 3 +++ nexus/db-queries/src/db/datastore/rack.rs | 9 +++++++ .../execution/src/external_networking.rs | 3 +++ .../planning/src/blueprint_builder/builder.rs | 1 + nexus/test-utils/src/lib.rs | 2 ++ nexus/tests/integration_tests/endpoints.rs | 1 + nexus/tests/integration_tests/instances.rs | 3 +++ nexus/types/src/external_api/params.rs | 3 +++ openapi/nexus-internal.json | 7 +++++ openapi/nexus.json | 27 ++++++++++++++++++- openapi/sled-agent.json | 7 +++++ schema/crdb/dbinit.sql | 14 +++++++--- schema/crdb/nic-spoof-allow/up01.sql | 2 ++ schema/crdb/nic-spoof-allow/up02.sql | 1 + schema/crdb/nic-spoof-allow/up03.sql | 20 ++++++++++++++ sled-agent/src/rack_setup/plan/service.rs | 3 +++ sled-agent/src/sim/server.rs | 2 ++ 24 files changed, 153 insertions(+), 8 deletions(-) create mode 100644 schema/crdb/nic-spoof-allow/up01.sql create mode 100644 schema/crdb/nic-spoof-allow/up02.sql create mode 100644 schema/crdb/nic-spoof-allow/up03.sql diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index f385c0b4fa..4aa57ff0da 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -2328,6 +2328,10 @@ pub struct InstanceNetworkInterface { /// True if this interface is the primary for the instance to which it's /// attached. pub primary: bool, + + /// A set of additional networks that this interface may send and + /// receive traffic on. + pub transit_ips: Vec, } #[derive( diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 51b829e214..0215d5a9d0 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -57,6 +57,7 @@ pub struct NetworkInterface { pub vni: external::Vni, pub primary: bool, pub slot: u8, + pub transit_ips: Vec, } /// An IP address and port range used for source NAT, i.e., making diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index caeda81217..353d25858a 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -440,6 +440,22 @@ impl PortManager { } } + // If there are any transit IPs set, allow them through. + // TODO: Currently set only in initial state. + // This, external IPs, and cfg'able state + // (DHCP?) are probably worth being managed by an RPW. + for block in &nic.transit_ips { + #[cfg(target_os = "illumos")] + hdl.allow_cidr(&port_name, super::net_to_cidr(*block)); + + debug!( + self.inner.log, + "Added CIDR to in/out allowlist"; + "port_name" => &port_name, + "cidr" => ?block, + ); + } + info!( self.inner.log, "Created OPTE port"; diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs index 8520afdb76..5409ad66b3 100644 --- a/nexus/db-model/src/network_interface.rs +++ b/nexus/db-model/src/network_interface.rs @@ -13,6 +13,7 @@ use chrono::DateTime; use chrono::Utc; use db_macros::Resource; use diesel::AsChangeset; +use ipnetwork::IpNetwork; use ipnetwork::NetworkSize; use nexus_types::external_api::params; use nexus_types::identity::Resource; @@ -63,11 +64,13 @@ pub struct NetworkInterface { // // If user requests an address of either kind, give exactly that and not the other. // If neither is specified, auto-assign one of each? - pub ip: ipnetwork::IpNetwork, + pub ip: IpNetwork, pub slot: SqlU8, #[diesel(column_name = is_primary)] pub primary: bool, + + pub transit_ips: Vec, } impl NetworkInterface { @@ -101,6 +104,7 @@ impl NetworkInterface { vni: external::Vni::try_from(0).unwrap(), primary: self.primary, slot: *self.slot, + transit_ips: self.transit_ips.into_iter().map(Into::into).collect(), } } } @@ -121,11 +125,13 @@ pub struct InstanceNetworkInterface { pub subnet_id: Uuid, pub mac: MacAddr, - pub ip: ipnetwork::IpNetwork, + pub ip: IpNetwork, pub slot: SqlU8, #[diesel(column_name = is_primary)] pub primary: bool, + + pub transit_ips: Vec, } /// Service Network Interface DB model. @@ -144,7 +150,7 @@ pub struct ServiceNetworkInterface { pub subnet_id: Uuid, pub mac: MacAddr, - pub ip: ipnetwork::IpNetwork, + pub ip: IpNetwork, pub slot: SqlU8, #[diesel(column_name = is_primary)] @@ -241,6 +247,7 @@ impl NetworkInterface { ip: self.ip, slot: self.slot, primary: self.primary, + transit_ips: self.transit_ips, } } @@ -289,6 +296,7 @@ impl From for NetworkInterface { ip: iface.ip, slot: iface.slot, primary: iface.primary, + transit_ips: iface.transit_ips, } } } @@ -312,6 +320,7 @@ impl From for NetworkInterface { ip: iface.ip, slot: iface.slot, primary: iface.primary, + transit_ips: vec![], } } } @@ -459,6 +468,7 @@ pub struct NetworkInterfaceUpdate { pub time_modified: DateTime, #[diesel(column_name = is_primary)] pub primary: Option, + pub transit_ips: Vec, } impl From for external::InstanceNetworkInterface { @@ -471,6 +481,11 @@ impl From for external::InstanceNetworkInterface { ip: iface.ip.ip(), mac: *iface.mac, primary: iface.primary, + transit_ips: iface + .transit_ips + .into_iter() + .map(Into::into) + .collect(), } } } @@ -483,6 +498,11 @@ impl From for NetworkInterfaceUpdate { description: params.identity.description, time_modified: Utc::now(), primary, + transit_ips: params + .transit_ips + .into_iter() + .map(Into::into) + .collect(), } } } diff --git a/nexus/db-model/src/omicron_zone_config.rs b/nexus/db-model/src/omicron_zone_config.rs index c2258dba6c..3b18a749a7 100644 --- a/nexus/db-model/src/omicron_zone_config.rs +++ b/nexus/db-model/src/omicron_zone_config.rs @@ -659,6 +659,7 @@ impl OmicronZoneNic { vni: omicron_common::api::external::Vni::try_from(*self.vni) .context("parsing VNI")?, subnet: self.subnet.into(), + transit_ips: vec![], }) } } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index a96bca8ab9..a26b64f32b 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -519,6 +519,7 @@ table! { ip -> Inet, slot -> Int2, is_primary -> Bool, + transit_ips -> Array, } } @@ -537,6 +538,7 @@ table! { ip -> Inet, slot -> Int2, is_primary -> Bool, + transit_ips -> Array, } } joinable!(instance_network_interface -> instance (instance_id)); diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index db783af78b..bebf822e48 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(65, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(66, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(66, "nic-spoof-allow"), KnownVersion::new(65, "vpc-subnet-routing"), KnownVersion::new(64, "add-view-for-v2p-mappings"), KnownVersion::new(63, "remove-producer-base-route-column"), diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index c8e071684b..79389bc232 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -61,6 +61,7 @@ struct NicInfo { vni: db::model::Vni, primary: bool, slot: i16, + transit_ips: Vec, } impl From for omicron_common::api::internal::shared::NetworkInterface { @@ -93,6 +94,7 @@ impl From for omicron_common::api::internal::shared::NetworkInterface { vni: nic.vni.0, primary: nic.primary, slot: u8::try_from(nic.slot).unwrap(), + transit_ips: nic.transit_ips.iter().map(|v| (*v).into()).collect(), } } } @@ -503,6 +505,7 @@ impl DataStore { vpc::vni, network_interface::is_primary, network_interface::slot, + network_interface::transit_ips, )) .get_results_async::( &*self.pool_connection_authorized(opctx).await?, diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 04901c7785..5c7707777c 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1388,6 +1388,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, }, ), @@ -1417,6 +1418,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), @@ -1464,6 +1466,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, }, ), @@ -1493,6 +1496,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), @@ -1718,6 +1722,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, }, ), @@ -1750,6 +1755,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, }, ), @@ -1988,6 +1994,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, }, ), @@ -2093,6 +2100,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, }, ), @@ -2125,6 +2133,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, }, ), diff --git a/nexus/reconfigurator/execution/src/external_networking.rs b/nexus/reconfigurator/execution/src/external_networking.rs index cff912c137..6e23879cb0 100644 --- a/nexus/reconfigurator/execution/src/external_networking.rs +++ b/nexus/reconfigurator/execution/src/external_networking.rs @@ -500,6 +500,7 @@ mod tests { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }; let dns_id = OmicronZoneUuid::new_v4(); @@ -526,6 +527,7 @@ mod tests { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }; // Boundary NTP: @@ -555,6 +557,7 @@ mod tests { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }; Self { diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 45aea75473..c92925e3d6 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -709,6 +709,7 @@ impl<'a> BlueprintBuilder<'a> { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], } }; diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index a078ce2a61..b70e512db8 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -687,6 +687,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { slot: 0, subnet: (*NEXUS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, + transit_ips: vec![], }, }), }); @@ -1043,6 +1044,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { slot: 0, subnet: (*DNS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, + transit_ips: vec![], }, }, ), diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index a1c4fbb1cb..22d401d21e 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -463,6 +463,7 @@ pub static DEMO_INSTANCE_NIC_PUT: Lazy = description: Some(String::from("an updated description")), }, primary: false, + transit_ips: vec![], }); pub static DEMO_CERTIFICATE_NAME: Lazy = diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 41894392c1..41e2588f7c 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -2213,6 +2213,7 @@ async fn test_instance_update_network_interfaces( description: Some(new_description.clone()), }, primary: false, + transit_ips: vec![], }; // Verify we fail to update the NIC when the instance is running @@ -2289,6 +2290,7 @@ async fn test_instance_update_network_interfaces( description: None, }, primary: true, + transit_ips: vec![], }; let updated_primary_iface1 = NexusRequest::object_put( client, @@ -2383,6 +2385,7 @@ async fn test_instance_update_network_interfaces( description: None, }, primary: true, + transit_ips: vec![], }; let new_primary_iface = NexusRequest::object_put( client, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 38826051fc..9a04b02d1c 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -807,6 +807,9 @@ pub struct InstanceNetworkInterfaceUpdate { // for the instance, though not the name. #[serde(default)] pub primary: bool, + + /// TODO: describe + pub transit_ips: Vec, } // CERTIFICATES diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index c7d476994d..f6fa1a48d9 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -3470,6 +3470,12 @@ "subnet": { "$ref": "#/components/schemas/IpNet" }, + "transit_ips": { + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNet" + } + }, "vni": { "$ref": "#/components/schemas/Vni" } @@ -3483,6 +3489,7 @@ "primary", "slot", "subnet", + "transit_ips", "vni" ] }, diff --git a/openapi/nexus.json b/openapi/nexus.json index 91d267c435..a312db663f 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -14649,6 +14649,13 @@ "type": "string", "format": "date-time" }, + "transit_ips": { + "description": "A set of additional networks that this interface may send and receive traffic on.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNet" + } + }, "vpc_id": { "description": "The VPC to which the interface belongs.", "type": "string", @@ -14666,6 +14673,7 @@ "subnet_id", "time_created", "time_modified", + "transit_ips", "vpc_id" ] }, @@ -14807,8 +14815,18 @@ "description": "Make a secondary interface the instance's primary interface.\n\nIf applied to a secondary interface, that interface will become the primary on the next reboot of the instance. Note that this may have implications for routing between instances, as the new primary interface will be on a distinct subnet from the previous primary interface.\n\nNote that this can only be used to select a new primary interface for an instance. Requests to change the primary interface into a secondary will return an error.", "default": false, "type": "boolean" + }, + "transit_ips": { + "description": "TODO: describe", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNet" + } } - } + }, + "required": [ + "transit_ips" + ] }, "InstanceResultsPage": { "description": "A single page of results", @@ -15766,6 +15784,12 @@ "subnet": { "$ref": "#/components/schemas/IpNet" }, + "transit_ips": { + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNet" + } + }, "vni": { "$ref": "#/components/schemas/Vni" } @@ -15779,6 +15803,7 @@ "primary", "slot", "subnet", + "transit_ips", "vni" ] }, diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 5480f0bcf1..9673e2b239 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -3594,6 +3594,12 @@ "subnet": { "$ref": "#/components/schemas/IpNet" }, + "transit_ips": { + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNet" + } + }, "vni": { "$ref": "#/components/schemas/Vni" } @@ -3607,6 +3613,7 @@ "primary", "slot", "subnet", + "transit_ips", "vni" ] }, diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index f354ea2fec..3e0da75233 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -1434,7 +1434,14 @@ CREATE TABLE IF NOT EXISTS omicron.public.network_interface ( * The primary interface appears in DNS and its address is used for external * connectivity. */ - is_primary BOOL NOT NULL + is_primary BOOL NOT NULL, + + /* + * A supplementary list of addresses/CIDR blocks which a NIC is + * *allowed* to send/receive traffic on, in addition to its + * assigned address. + */ + transit_ips INET[] NOT NULL DEFAULT ARRAY[] ); /* A view of the network_interface table for just instance-kind records. */ @@ -1452,7 +1459,8 @@ SELECT mac, ip, slot, - is_primary + is_primary, + transit_ips FROM omicron.public.network_interface WHERE @@ -3935,7 +3943,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '65.0.0', NULL) + (TRUE, NOW(), NOW(), '66.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/nic-spoof-allow/up01.sql b/schema/crdb/nic-spoof-allow/up01.sql new file mode 100644 index 0000000000..2ca13e0a38 --- /dev/null +++ b/schema/crdb/nic-spoof-allow/up01.sql @@ -0,0 +1,2 @@ +ALTER TABLE omicron.public.network_interface +ADD COLUMN IF NOT EXISTS transit_ips INET[] NOT NULL DEFAULT ARRAY[]; diff --git a/schema/crdb/nic-spoof-allow/up02.sql b/schema/crdb/nic-spoof-allow/up02.sql new file mode 100644 index 0000000000..68ab39567d --- /dev/null +++ b/schema/crdb/nic-spoof-allow/up02.sql @@ -0,0 +1 @@ +DROP VIEW IF EXISTS omicron.public.instance_network_interface; diff --git a/schema/crdb/nic-spoof-allow/up03.sql b/schema/crdb/nic-spoof-allow/up03.sql new file mode 100644 index 0000000000..ac3cfe6b32 --- /dev/null +++ b/schema/crdb/nic-spoof-allow/up03.sql @@ -0,0 +1,20 @@ +CREATE VIEW IF NOT EXISTS omicron.public.instance_network_interface AS +SELECT + id, + name, + description, + time_created, + time_modified, + time_deleted, + parent_id AS instance_id, + vpc_id, + subnet_id, + mac, + ip, + slot, + is_primary, + transit_ips +FROM + omicron.public.network_interface +WHERE + kind = 'instance'; diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index b4a6fe76f6..2af0556d63 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -1055,6 +1055,7 @@ impl ServicePortBuilder { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }; Some((nic, external_ip)) @@ -1095,6 +1096,7 @@ impl ServicePortBuilder { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }; Ok((nic, external_ip)) @@ -1152,6 +1154,7 @@ impl ServicePortBuilder { vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }; Ok((nic, snat_cfg)) diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index ebee0adc1f..dcbfb8b80e 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -409,6 +409,7 @@ pub async fn run_standalone_server( vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, external_tls: false, external_dns_servers: vec![], @@ -452,6 +453,7 @@ pub async fn run_standalone_server( vni: Vni::SERVICES_VNI, primary: true, slot: 0, + transit_ips: vec![], }, }, }); From c1dc6d058229699ff8ce7bdd06349543007e46b7 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 29 May 2024 15:38:27 +0100 Subject: [PATCH 35/59] Test router attach swap, cleanup tests. --- nexus/db-model/src/collection.rs | 4 + nexus/db-model/src/vpc_router.rs | 2 + nexus/db-queries/src/db/collection_attach.rs | 26 +++- nexus/tests/integration_tests/vpc_routers.rs | 141 ++++++++++++------- nexus/types/src/external_api/params.rs | 9 +- 5 files changed, 123 insertions(+), 59 deletions(-) diff --git a/nexus/db-model/src/collection.rs b/nexus/db-model/src/collection.rs index b86e35d407..964aaad248 100644 --- a/nexus/db-model/src/collection.rs +++ b/nexus/db-model/src/collection.rs @@ -152,4 +152,8 @@ pub trait DatastoreAttachTargetConfig: type ResourceTimeDeletedColumn: Column::Table> + Default + ExpressionMethods; + + /// Controls whether a resource may be attached to a new collection without + /// first being explicitly detached from the previous one + const ALLOW_FROM_ATTACHED: bool = false; } diff --git a/nexus/db-model/src/vpc_router.rs b/nexus/db-model/src/vpc_router.rs index 9ceb6e3945..ee8988ae69 100644 --- a/nexus/db-model/src/vpc_router.rs +++ b/nexus/db-model/src/vpc_router.rs @@ -110,4 +110,6 @@ impl DatastoreAttachTargetConfig for VpcRouter { type ResourceIdColumn = vpc_subnet::dsl::id; type ResourceCollectionIdColumn = vpc_subnet::dsl::custom_router_id; type ResourceTimeDeletedColumn = vpc_subnet::dsl::time_deleted; + + const ALLOW_FROM_ATTACHED: bool = true; } diff --git a/nexus/db-queries/src/db/collection_attach.rs b/nexus/db-queries/src/db/collection_attach.rs index fccc1aa324..95e6afeb4b 100644 --- a/nexus/db-queries/src/db/collection_attach.rs +++ b/nexus/db-queries/src/db/collection_attach.rs @@ -232,12 +232,26 @@ pub trait DatastoreAttachTarget: .filter(collection_table().primary_key().eq(collection_id)) .filter(Self::CollectionTimeDeletedColumn::default().is_null()), ); - let resource_query = Box::new( - resource_query - .filter(resource_table().primary_key().eq(resource_id)) - .filter(Self::ResourceTimeDeletedColumn::default().is_null()) - .filter(Self::ResourceCollectionIdColumn::default().is_null()), - ); + let resource_query = if Self::ALLOW_FROM_ATTACHED { + Box::new( + resource_query + .filter(resource_table().primary_key().eq(resource_id)) + .filter( + Self::ResourceTimeDeletedColumn::default().is_null(), + ), + ) + } else { + Box::new( + resource_query + .filter(resource_table().primary_key().eq(resource_id)) + .filter( + Self::ResourceTimeDeletedColumn::default().is_null(), + ) + .filter( + Self::ResourceCollectionIdColumn::default().is_null(), + ), + ) + }; let update_resource_statement = update .into_boxed() diff --git a/nexus/tests/integration_tests/vpc_routers.rs b/nexus/tests/integration_tests/vpc_routers.rs index d59dad4afd..ab9d620d3e 100644 --- a/nexus/tests/integration_tests/vpc_routers.rs +++ b/nexus/tests/integration_tests/vpc_routers.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use dropshot::test_util::ClientTestContext; use http::method::Method; use http::StatusCode; use nexus_test_utils::http_testing::AuthnMode; @@ -23,6 +24,9 @@ use nexus_types::external_api::views::VpcSubnet; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::IdentityMetadataUpdateParams; use omicron_common::api::external::Ipv4Net; +use omicron_common::api::external::NameOrId; + +const PROJECT_NAME: &str = "os-cartographers"; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -32,15 +36,14 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; // Create a project that we'll use for testing. - let project_name = "springfield-squidport"; - let _ = create_project(&client, project_name).await; + let _ = create_project(&client, PROJECT_NAME).await; // Create a VPC. let vpc_name = "vpc1"; - let vpc = create_vpc(&client, project_name, vpc_name).await; + let vpc = create_vpc(&client, PROJECT_NAME, vpc_name).await; let routers_url = - format!("/v1/vpc-routers?project={}&vpc={}", project_name, vpc_name); + format!("/v1/vpc-routers?project={}&vpc={}", PROJECT_NAME, vpc_name); // get routers should have only the system router created w/ the VPC let routers = @@ -51,7 +54,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { let router_name = "router1"; let router_url = format!( "/v1/vpc-routers/{}?project={}&vpc={}", - router_name, project_name, vpc_name + router_name, PROJECT_NAME, vpc_name ); // fetching a particular router should 404 @@ -71,7 +74,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { // Create a VPC Router. let router = - create_router(&client, project_name, vpc_name, router_name).await; + create_router(&client, PROJECT_NAME, vpc_name, router_name).await; assert_eq!(router.identity.name, router_name); assert_eq!(router.identity.description, "router description"); assert_eq!(router.vpc_id, vpc.identity.id); @@ -114,7 +117,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { let router2_name = "router2"; let router2_url = format!( "/v1/vpc-routers/{}?project={}&vpc={}", - router2_name, project_name, vpc_name + router2_name, PROJECT_NAME, vpc_name ); // second router 404s before it's created @@ -134,7 +137,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { // create second custom router let router2 = - create_router(client, project_name, vpc_name, router2_name).await; + create_router(client, PROJECT_NAME, vpc_name, router2_name).await; assert_eq!(router2.identity.name, router2_name); assert_eq!(router2.vpc_id, vpc.identity.id); assert_eq!(router2.kind, VpcRouterKind::Custom); @@ -185,7 +188,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { let router_url = format!( "/v1/vpc-routers/new-name?project={}&vpc={}", - project_name, vpc_name + PROJECT_NAME, vpc_name ); // fetching by new name works @@ -251,10 +254,10 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { // Creating a router with the same name in a different VPC is allowed let vpc2_name = "vpc2"; - let vpc2 = create_vpc(&client, project_name, vpc2_name).await; + let vpc2 = create_vpc(&client, PROJECT_NAME, vpc2_name).await; let router_same_name = - create_router(&client, project_name, vpc2_name, router2_name).await; + create_router(&client, PROJECT_NAME, vpc2_name, router2_name).await; assert_eq!(router_same_name.identity.name, router2_name); assert_eq!(router_same_name.vpc_id, vpc2.identity.id); } @@ -272,14 +275,14 @@ async fn test_vpc_routers_attach_to_subnet( // Create a project that we'll use for testing. // This includes the vpc 'default'. - let project_name = "springfield-squidport"; - let _ = create_project(&client, project_name).await; + let _ = create_project(&client, PROJECT_NAME).await; let vpc_name = "default"; + let subnet_name = "default"; let routers_url = - format!("/v1/vpc-routers?project={}&vpc={}", project_name, vpc_name); + format!("/v1/vpc-routers?project={}&vpc={}", PROJECT_NAME, vpc_name); let subnets_url = - format!("/v1/vpc-subnets?project={}&vpc={}", project_name, vpc_name); + format!("/v1/vpc-subnets?project={}&vpc={}", PROJECT_NAME, vpc_name); // get routers should have only the system router created w/ the VPC let routers = @@ -293,14 +296,14 @@ async fn test_vpc_routers_attach_to_subnet( // Create a custom router for later use. let router_name = "routy"; let router = - create_router(&client, project_name, vpc_name, router_name).await; + create_router(&client, PROJECT_NAME, vpc_name, router_name).await; assert_eq!(router.kind, VpcRouterKind::Custom); // Attaching a system router should fail. let err = object_put_error( client, &format!( - "/v1/vpc-subnets/default?project={project_name}&vpc={vpc_name}" + "/v1/vpc-subnets/{subnet_name}?project={PROJECT_NAME}&vpc={vpc_name}" ), &VpcSubnetUpdate { identity: IdentityMetadataUpdateParams { @@ -315,30 +318,23 @@ async fn test_vpc_routers_attach_to_subnet( assert_eq!(err.message, "cannot attach a system router to a VPC subnet"); // Attaching a new custom router should succeed. - let default_subnet: VpcSubnet = object_put( + let default_subnet = set_custom_router( client, - &format!( - "/v1/vpc-subnets/default?project={project_name}&vpc={vpc_name}" - ), - &VpcSubnetUpdate { - identity: IdentityMetadataUpdateParams { - name: None, - description: None, - }, - custom_router: Some(router.identity.id.into()), - }, + "default", + vpc_name, + Some(router.identity.id.into()), ) .await; assert_eq!(default_subnet.custom_router_id, Some(router.identity.id)); // Attaching a custom router to another subnet (same VPC) should succeed: // ... at create time. - let subnet_name = "subnetty"; + let subnet2_name = "subnetty"; let subnet2 = create_vpc_subnet( &client, - &project_name, + &PROJECT_NAME, &vpc_name, - &subnet_name, + &subnet2_name, Ipv4Net("192.168.0.0/24".parse().unwrap()), None, Some(router_name), @@ -347,40 +343,32 @@ async fn test_vpc_routers_attach_to_subnet( assert_eq!(subnet2.custom_router_id, Some(router.identity.id)); // ... and via update. - let subnet_name = "subnettier"; + let subnet3_name = "subnettier"; let _ = create_vpc_subnet( &client, - &project_name, + &PROJECT_NAME, &vpc_name, - &subnet_name, + &subnet3_name, Ipv4Net("192.168.1.0/24".parse().unwrap()), None, None, ) .await; - let subnet3: VpcSubnet = object_put( + let subnet3 = set_custom_router( client, - &format!( - "/v1/vpc-subnets/{subnet_name}?project={project_name}&vpc={vpc_name}", - ), - &VpcSubnetUpdate { - identity: IdentityMetadataUpdateParams { - name: None, - description: None, - }, - custom_router: Some(router.identity.id.into()), - }, + subnet3_name, + vpc_name, + Some(router.identity.id.into()), ) .await; - assert_eq!(subnet3.custom_router_id, Some(router.identity.id)); // Attaching a custom router to another VPC's subnet should fail. - create_vpc(&client, project_name, "vpc1").await; + create_vpc(&client, PROJECT_NAME, "vpc1").await; let err = object_put_error( client, - &format!("/v1/vpc-subnets/default?project={project_name}&vpc=vpc1"), + &format!("/v1/vpc-subnets/default?project={PROJECT_NAME}&vpc=vpc1"), &VpcSubnetUpdate { identity: IdentityMetadataUpdateParams { name: None, @@ -393,11 +381,40 @@ async fn test_vpc_routers_attach_to_subnet( .await; assert_eq!(err.message, "router and subnet must belong to the same VPC"); - // Deleting a custom router should detach from all these subnets. + // Detach (and double detach) should succeed without issue. + let subnet3 = set_custom_router(client, subnet3_name, vpc_name, None).await; + assert_eq!(subnet3.custom_router_id, None); + let subnet3 = set_custom_router(client, subnet3_name, vpc_name, None).await; + assert_eq!(subnet3.custom_router_id, None); + + // Assigning a new router should not require that we first detach the old one. + let router2_name = "routier"; + let router2 = + create_router(&client, PROJECT_NAME, vpc_name, router2_name).await; + let subnet2 = set_custom_router( + client, + subnet2_name, + vpc_name, + Some(router2.identity.id.into()), + ) + .await; + assert_eq!(subnet2.custom_router_id, Some(router2.identity.id)); + + // Reset subnet2 back to our first router. + let subnet2 = set_custom_router( + client, + subnet2_name, + vpc_name, + Some(router.identity.id.into()), + ) + .await; + assert_eq!(subnet2.custom_router_id, Some(router.identity.id)); + + // Deleting a custom router should detach from remaining subnets. object_delete( &client, &format!( - "/v1/vpc-routers/{router_name}?vpc={}&project={project_name}", + "/v1/vpc-routers/{router_name}?vpc={}&project={PROJECT_NAME}", "default" ), ) @@ -406,7 +423,7 @@ async fn test_vpc_routers_attach_to_subnet( for subnet in objects_list_page_authz::(client, &subnets_url).await.items { - assert!(subnet.custom_router_id.is_none()); + assert!(subnet.custom_router_id.is_none(), "{subnet:?}"); } } @@ -470,6 +487,28 @@ async fn test_vpc_routers_disallowed_custom_targets( todo!() } +async fn set_custom_router( + client: &ClientTestContext, + subnet_name: &str, + vpc_name: &str, + custom_router: Option, +) -> VpcSubnet { + object_put( + client, + &format!( + "/v1/vpc-subnets/{subnet_name}?project={PROJECT_NAME}&vpc={vpc_name}" + ), + &VpcSubnetUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: None, + }, + custom_router, + }, + ) + .await +} + fn routers_eq(sn1: &VpcRouter, sn2: &VpcRouter) { identity_eq(&sn1.identity, &sn2.identity); assert_eq!(sn1.vpc_id, sn2.vpc_id); diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 9a04b02d1c..a003e2204b 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -808,7 +808,8 @@ pub struct InstanceNetworkInterfaceUpdate { #[serde(default)] pub primary: bool, - /// TODO: describe + /// A set of additional networks that this interface may send and + /// receive traffic on. pub transit_ips: Vec, } @@ -1237,7 +1238,11 @@ pub struct VpcSubnetUpdate { #[serde(flatten)] pub identity: IdentityMetadataUpdateParams, - /// XXX + /// An optional router, used to direct packets sent from hosts in this subnet + /// to any destination address. + /// + /// Custom routers apply in addition to the VPC-wide *system* router, and have + /// higher priority than pub custom_router: Option, } From 0f0dcb65130af359196d2c73283ad20e00a37422 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 29 May 2024 17:01:42 +0100 Subject: [PATCH 36/59] Validate custom, dest/target pairs on route create --- nexus/src/app/vpc_router.rs | 47 +++++++++- nexus/test-utils/src/resource_helpers.rs | 75 +++++++++++++++ .../tests/integration_tests/router_routes.rs | 91 ++++++++++++++++++- nexus/tests/integration_tests/vpc_routers.rs | 53 +++++------ 4 files changed, 234 insertions(+), 32 deletions(-) diff --git a/nexus/src/app/vpc_router.rs b/nexus/src/app/vpc_router.rs index 40b4c1de0f..8b18a77c07 100644 --- a/nexus/src/app/vpc_router.rs +++ b/nexus/src/app/vpc_router.rs @@ -17,11 +17,15 @@ use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; +use omicron_common::api::external::IpNet; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::NameOrId; +use omicron_common::api::external::RouteDestination; +use omicron_common::api::external::RouteTarget; use omicron_common::api::external::RouterRouteKind; use omicron_common::api::external::UpdateResult; +use std::net::IpAddr; use uuid::Uuid; impl super::Nexus { @@ -191,8 +195,47 @@ impl super::Nexus { kind: &RouterRouteKind, params: ¶ms::RouterRouteCreate, ) -> CreateResult { - let (.., authz_router) = - router_lookup.lookup_for(authz::Action::CreateChild).await?; + let (.., authz_router, db_router) = + router_lookup.fetch_for(authz::Action::CreateChild).await?; + + if db_router.kind == VpcRouterKind::System { + return Err(Error::invalid_request( + "user-provided routes cannot be added to a system router", + )); + } + + // Validate route destinations/targets at this stage: + // - mixed explicit v4 and v6 are disallowed. + // - users cannot specify 'Vpc' as a custom router dest/target. + // - users cannot specify 'Subnet' as a custom router target. + // - the only internet gateway we support today is 'outbound'. + match (¶ms.destination, ¶ms.target) { + (RouteDestination::Ip(IpAddr::V4(_)), RouteTarget::Ip(IpAddr::V4(_))) + | (RouteDestination::Ip(IpAddr::V6(_)), RouteTarget::Ip(IpAddr::V6(_))) + | (RouteDestination::IpNet(IpNet::V4(_)), RouteTarget::Ip(IpAddr::V4(_))) + | (RouteDestination::IpNet(IpNet::V6(_)), RouteTarget::Ip(IpAddr::V6(_))) => {}, + + (RouteDestination::Ip(_), RouteTarget::Ip(_)) + | (RouteDestination::IpNet(_), RouteTarget::Ip(_)) + => return Err(Error::invalid_request( + "cannot mix explicit IPv4 and IPv6 addresses between destination and target" + )), + + (RouteDestination::Vpc(_), _) | (_, RouteTarget::Vpc(_)) => return Err(Error::invalid_request( + "VPCs cannot be used as a destination or target in custom routers" + )), + + (_, RouteTarget::Subnet(_)) => return Err(Error::invalid_request( + "subnets cannot be used as a target in custom routers" + )), + + (_, RouteTarget::InternetGateway(n)) if n.as_str() != "outbound" => return Err(Error::invalid_request( + "'outbound' is currently the only valid internet gateway" + )), + + _ => {}, + }; + let id = Uuid::new_v4(); let route = db::model::RouterRoute::new( id, diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 57e1113a91..1185c8b9b4 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -39,6 +39,9 @@ use omicron_common::api::external::InstanceCpuCount; use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::Ipv6Net; use omicron_common::api::external::NameOrId; +use omicron_common::api::external::RouteDestination; +use omicron_common::api::external::RouteTarget; +use omicron_common::api::external::RouterRoute; use omicron_common::disk::DiskIdentity; use omicron_sled_agent::sim::SledAgent; use omicron_test_utils::dev::poll::wait_for_condition; @@ -613,6 +616,78 @@ pub async fn create_router( .unwrap() } +pub async fn create_route( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + router_name: &str, + route_name: &str, + destination: RouteDestination, + target: RouteTarget, +) -> RouterRoute { + NexusRequest::objects_post( + &client, + format!( + "/v1/vpc-router-routes?project={}&vpc={}&router={}", + &project_name, &vpc_name, &router_name + ) + .as_str(), + ¶ms::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: route_name.parse().unwrap(), + description: String::from("route description"), + }, + target, + destination, + }, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + +#[allow(clippy::too_many_arguments)] +pub async fn create_route_with_error( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + router_name: &str, + route_name: &str, + destination: RouteDestination, + target: RouteTarget, + status: StatusCode, +) -> HttpErrorResponseBody { + NexusRequest::new( + RequestBuilder::new( + client, + Method::POST, + format!( + "/v1/vpc-router-routes?project={}&vpc={}&router={}", + &project_name, &vpc_name, &router_name + ) + .as_str(), + ) + .body(Some(¶ms::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: route_name.parse().unwrap(), + description: String::from("route description"), + }, + target, + destination, + })) + .expect_status(Some(status)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + pub async fn assert_ip_pool_utilization( client: &ClientTestContext, pool_name: &str, diff --git a/nexus/tests/integration_tests/router_routes.rs b/nexus/tests/integration_tests/router_routes.rs index a13026a7fd..aa5a6f50c1 100644 --- a/nexus/tests/integration_tests/router_routes.rs +++ b/nexus/tests/integration_tests/router_routes.rs @@ -4,9 +4,12 @@ use dropshot::Method; use http::StatusCode; +use itertools::Itertools; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::identity_eq; +use nexus_test_utils::resource_helpers::create_route; +use nexus_test_utils::resource_helpers::create_route_with_error; use nexus_test_utils::resource_helpers::objects_list_page_authz; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; @@ -23,11 +26,15 @@ use nexus_test_utils::resource_helpers::{ create_project, create_router, create_vpc, }; +use crate::integration_tests::vpc_routers::PROJECT_NAME; + type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; #[nexus_test] -async fn test_router_routes(cptestctx: &ControlPlaneTestContext) { +async fn test_router_routes_crud_operations( + cptestctx: &ControlPlaneTestContext, +) { let client = &cptestctx.external_client; let project_name = "springfield-squidport"; @@ -212,3 +219,85 @@ async fn test_router_routes(cptestctx: &ControlPlaneTestContext) { .await .unwrap(); } + +#[nexus_test] +async fn test_router_routes_disallow_mixed_v4_v6( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + let _ = create_project(&client, PROJECT_NAME).await; + + let vpc_name = "default"; + let router_name = "routy"; + let _router = + create_router(&client, PROJECT_NAME, vpc_name, router_name).await; + + // Some targets/strings refer to a mixed v4/v6 entity, e.g., + // subnet or instance. Others refer to one kind only (ipnet, ip). + // Users should not be able to mix v4 and v6 in these latter routes + // -- route resolution will ignore them, but a helpful error message + // is more useful. + let dest_set: [RouteDestination; 5] = [ + "ip:4.4.4.4".parse().unwrap(), + "ipnet:4.4.4.0/24".parse().unwrap(), + "ip:2001:4860:4860::8888".parse().unwrap(), + "ipnet:2001:4860:4860::/64".parse().unwrap(), + "subnet:named-subnet".parse().unwrap(), + ]; + + let target_set: [RouteTarget; 5] = [ + "ip:172.30.0.5".parse().unwrap(), + "ip:fd37:faf4:cc25::5".parse().unwrap(), + "instance:named-instance".parse().unwrap(), + "inetgw:outbound".parse().unwrap(), + "drop".parse().unwrap(), + ]; + + for (i, (dest, target)) in dest_set + .into_iter() + .cartesian_product(target_set.into_iter()) + .enumerate() + { + use RouteDestination as Rd; + use RouteTarget as Rt; + let allowed = match (&dest, &target) { + (Rd::Ip(IpAddr::V4(_)), Rt::Ip(IpAddr::V4(_))) + | (Rd::Ip(IpAddr::V6(_)), Rt::Ip(IpAddr::V6(_))) + | (Rd::IpNet(IpNet::V4(_)), Rt::Ip(IpAddr::V4(_))) + | (Rd::IpNet(IpNet::V6(_)), Rt::Ip(IpAddr::V6(_))) => true, + (Rd::Ip(_), Rt::Ip(_)) | (Rd::IpNet(_), Rt::Ip(_)) => false, + _ => true, + }; + + let route_name = format!("test-route-{i}"); + + if allowed { + create_route( + client, + PROJECT_NAME, + vpc_name, + router_name, + &route_name, + dest, + target, + ) + .await; + } else { + let err = create_route_with_error( + client, + PROJECT_NAME, + vpc_name, + router_name, + &route_name, + dest, + target, + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + err.message, + "cannot mix explicit IPv4 and IPv6 addresses between destination and target" + ); + } + } +} diff --git a/nexus/tests/integration_tests/vpc_routers.rs b/nexus/tests/integration_tests/vpc_routers.rs index ab9d620d3e..8d6bd14bf7 100644 --- a/nexus/tests/integration_tests/vpc_routers.rs +++ b/nexus/tests/integration_tests/vpc_routers.rs @@ -26,13 +26,13 @@ use omicron_common::api::external::IdentityMetadataUpdateParams; use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::NameOrId; -const PROJECT_NAME: &str = "os-cartographers"; +pub const PROJECT_NAME: &str = "os-cartographers"; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; #[nexus_test] -async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { +async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; // Create a project that we'll use for testing. @@ -46,8 +46,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { format!("/v1/vpc-routers?project={}&vpc={}", PROJECT_NAME, vpc_name); // get routers should have only the system router created w/ the VPC - let routers = - objects_list_page_authz::(client, &routers_url).await.items; + let routers = list_routers(client, &vpc_name).await; assert_eq!(routers.len(), 1); assert_eq!(routers[0].kind, VpcRouterKind::System); @@ -91,7 +90,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { routers_eq(&router, &same_router); // routers list should now have the one in it - let routers = objects_list_page_authz(client, &routers_url).await.items; + let routers = list_routers(client, &vpc_name).await; assert_eq!(routers.len(), 2); routers_eq(&routers[0], &router); @@ -143,8 +142,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { assert_eq!(router2.kind, VpcRouterKind::Custom); // routers list should now have two custom and one system - let routers = - objects_list_page_authz::(client, &routers_url).await.items; + let routers = list_routers(client, &vpc_name).await; assert_eq!(routers.len(), 3); routers_eq(&routers[0], &router); routers_eq(&routers[1], &router2); @@ -204,8 +202,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { assert_eq!(&updated_router.identity.description, "another description"); // fetching list should show updated one - let routers = - objects_list_page_authz::(client, &routers_url).await.items; + let routers = list_routers(client, &vpc_name).await; assert_eq!(routers.len(), 3); routers_eq(&routers[0], &updated_router); @@ -217,8 +214,7 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { .unwrap(); // routers list should now have two again, one system and one custom - let routers = - objects_list_page_authz::(client, &routers_url).await.items; + let routers = list_routers(client, &vpc_name).await; assert_eq!(routers.len(), 2); routers_eq(&routers[0], &router2); @@ -266,32 +262,22 @@ async fn test_vpc_routers(cptestctx: &ControlPlaneTestContext) { async fn test_vpc_routers_attach_to_subnet( cptestctx: &ControlPlaneTestContext, ) { - // XXX: really clean this up. let client = &cptestctx.external_client; - // --- - // XX: copied from above - // - // Create a project that we'll use for testing. // This includes the vpc 'default'. let _ = create_project(&client, PROJECT_NAME).await; + let vpc_name = "default"; let subnet_name = "default"; - let routers_url = - format!("/v1/vpc-routers?project={}&vpc={}", PROJECT_NAME, vpc_name); let subnets_url = format!("/v1/vpc-subnets?project={}&vpc={}", PROJECT_NAME, vpc_name); // get routers should have only the system router created w/ the VPC - let routers = - objects_list_page_authz::(client, &routers_url).await.items; + let routers = list_routers(client, vpc_name).await; assert_eq!(routers.len(), 1); assert_eq!(routers[0].kind, VpcRouterKind::System); - // - // XX: copied from above - // --- // Create a custom router for later use. let router_name = "routy"; @@ -433,13 +419,12 @@ async fn test_vpc_routers_custom_route_at_instance( ) { let _client = &cptestctx.external_client; - // Attempting to delete a system router should fail. + // Installing a custom router onto a subnet with a live instance + // should install routes at that sled. - // Attempting to add a new route to a system router should fail. + // Swapping router should change the installed routes at that sled. - // Attempting to modify/delete a VPC subnet route should fail. - - // Modifying the target of a Default (gateway) route should succeed. + // Unsetting a router should remove affected non-system routes. todo!() } @@ -476,7 +461,7 @@ async fn test_vpc_routers_internet_gateway_target( } #[nexus_test] -async fn test_vpc_routers_disallowed_custom_targets( +async fn test_vpc_routers_disallow_custom_targets( cptestctx: &ControlPlaneTestContext, ) { let _client = &cptestctx.external_client; @@ -509,6 +494,16 @@ async fn set_custom_router( .await } +async fn list_routers( + client: &ClientTestContext, + vpc_name: &str, +) -> Vec { + let routers_url = + format!("/v1/vpc-routers?project={}&vpc={}", PROJECT_NAME, vpc_name); + let out = objects_list_page_authz::(client, &routers_url).await; + out.items +} + fn routers_eq(sn1: &VpcRouter, sn2: &VpcRouter) { identity_eq(&sn1.identity, &sn2.identity); assert_eq!(sn1.vpc_id, sn2.vpc_id); From 3d179607a4b90f1d8af4aa2302b808f8ff5757c5 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 29 May 2024 20:00:52 +0100 Subject: [PATCH 37/59] Visit + test allowed routes and modification semantics in-depth. --- nexus/db-queries/src/db/datastore/vpc.rs | 7 +- nexus/src/app/vpc_router.rs | 24 +- .../tests/integration_tests/router_routes.rs | 303 ++++++++++++++++-- nexus/tests/integration_tests/vpc_routers.rs | 62 ++-- nexus/types/src/external_api/params.rs | 6 +- openapi/nexus.json | 6 +- 6 files changed, 313 insertions(+), 95 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index b1e5097d24..d243c19ee4 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1141,9 +1141,10 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; // Unlink all subnets from this router. - // XXX: We might this want to error out before the delete fires. - // XXX: This will temporarily leave some hanging subnet attachments. We need to be sure - // these are safely handled (no unwrap, or remove via join here.) + // This will temporarily leave some hanging subnet attachments. + // `vpc_get_active_custom_routers` will join and then filter, + // so such rows will be treated as though they have no custom router + // by the RPW. use db::schema::vpc_subnet::dsl as vpc; diesel::update(vpc::vpc_subnet) .filter(vpc::time_deleted.is_null()) diff --git a/nexus/src/app/vpc_router.rs b/nexus/src/app/vpc_router.rs index 8b18a77c07..1cb2bd3dda 100644 --- a/nexus/src/app/vpc_router.rs +++ b/nexus/src/app/vpc_router.rs @@ -134,7 +134,7 @@ impl super::Nexus { // router kind cannot be changed, but it might be able to save us a // database round-trip. if db_router.kind == VpcRouterKind::System { - return Err(Error::invalid_request("Cannot delete system router")); + return Err(Error::invalid_request("cannot delete system router")); } let out = self.db_datastore.vpc_delete_router(opctx, &authz_router).await?; @@ -272,21 +272,31 @@ impl super::Nexus { route_lookup: &lookup::RouterRoute<'_>, params: ¶ms::RouterRouteUpdate, ) -> UpdateResult { - let (.., vpc, authz_router, authz_route, db_route) = + let (.., authz_router, authz_route, db_route) = route_lookup.fetch_for(authz::Action::Modify).await?; - // TODO: Write a test for this once there's a way to test it (i.e. - // subnets automatically register to the system router table) + match db_route.kind.0 { - RouterRouteKind::Custom | RouterRouteKind::Default => (), + // Default routes allow a constrained form of modification: + // only the target may change. + RouterRouteKind::Default if + params.identity.name.is_some() + || params.identity.description.is_some() + || params.destination != db_route.destination.0 => { + return Err(Error::invalid_request( + "the destination and metadata of a Default route cannot be changed", + ))}, + + RouterRouteKind::Custom | RouterRouteKind::Default => {}, + _ => { return Err(Error::invalid_request(format!( - "routes of type {} from the system table of VPC {:?} \ + "routes of type {} within the system router \ are not modifiable", db_route.kind.0, - vpc.id() ))); } } + let out = self .db_datastore .router_update_route(&opctx, &authz_route, params.clone().into()) diff --git a/nexus/tests/integration_tests/router_routes.rs b/nexus/tests/integration_tests/router_routes.rs index aa5a6f50c1..a0aaa596c7 100644 --- a/nexus/tests/integration_tests/router_routes.rs +++ b/nexus/tests/integration_tests/router_routes.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use dropshot::test_util::ClientTestContext; use dropshot::Method; use http::StatusCode; use itertools::Itertools; @@ -10,9 +11,12 @@ use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::identity_eq; use nexus_test_utils::resource_helpers::create_route; use nexus_test_utils::resource_helpers::create_route_with_error; +use nexus_test_utils::resource_helpers::object_put; +use nexus_test_utils::resource_helpers::object_put_error; use nexus_test_utils::resource_helpers::objects_list_page_authz; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; +use nexus_types::external_api::params::RouterRouteUpdate; use omicron_common::api::external::IpNet; use omicron_common::api::external::SimpleIdentity; use omicron_common::api::external::{ @@ -31,39 +35,32 @@ use crate::integration_tests::vpc_routers::PROJECT_NAME; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; -#[nexus_test] -async fn test_router_routes_crud_operations( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - - let project_name = "springfield-squidport"; - let vpc_name = "vpc1"; - let router_name = "router1"; - - let get_routes_url = |router_name: &str| -> String { - format!( - "/v1/vpc-router-routes?project={}&vpc={}&router={}", - project_name, vpc_name, router_name - ) - }; - - let get_route_url = |router_name: &str, route_name: &str| -> String { - format!( - "/v1/vpc-router-routes/{}?project={}&vpc={}&router={}", - route_name, project_name, vpc_name, router_name - ) - }; - - let _ = create_project(&client, project_name).await; +fn get_routes_url(vpc_name: &str, router_name: &str) -> String { + format!( + "/v1/vpc-router-routes?project={}&vpc={}&router={}", + PROJECT_NAME, vpc_name, router_name + ) +} - // Create a vpc - create_vpc(&client, project_name, vpc_name).await; +fn get_route_url( + vpc_name: &str, + router_name: &str, + route_name: &str, +) -> String { + format!( + "/v1/vpc-router-routes/{}?project={}&vpc={}&router={}", + route_name, PROJECT_NAME, vpc_name, router_name + ) +} +async fn get_system_routes( + client: &ClientTestContext, + vpc_name: &str, +) -> [RouterRoute; 3] { // Get the system router's routes let system_router_routes = objects_list_page_authz::( client, - get_routes_url("system").as_str(), + get_routes_url(vpc_name, "system").as_str(), ) .await .items; @@ -93,6 +90,27 @@ async fn test_router_routes_crud_operations( let subnet_route = subnet_route.expect("no default subnet route found in system router"); + [v4_route, v6_route, subnet_route] +} + +#[nexus_test] +async fn test_router_routes_crud_operations( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + let vpc_name = "vpc1"; + let router_name = "router1"; + + let _ = create_project(&client, PROJECT_NAME).await; + + // Create a vpc + create_vpc(&client, PROJECT_NAME, vpc_name).await; + + // Get the system router's routes + let [v4_route, v6_route, subnet_route] = + get_system_routes(client, vpc_name).await; + // Deleting any default system route is disallowed. for route in &[&v4_route, &v6_route, &subnet_route] { let error: dropshot::HttpErrorResponseBody = @@ -100,7 +118,8 @@ async fn test_router_routes_crud_operations( client, StatusCode::BAD_REQUEST, Method::DELETE, - get_route_url("system", route.name().as_str()).as_str(), + get_route_url(vpc_name, "system", route.name().as_str()) + .as_str(), ) .authn_as(AuthnMode::PrivilegedUser) .execute() @@ -112,12 +131,12 @@ async fn test_router_routes_crud_operations( } // Create a custom router - create_router(&client, project_name, vpc_name, router_name).await; + create_router(&client, PROJECT_NAME, vpc_name, router_name).await; // Get routes list for custom router let routes = objects_list_page_authz::( client, - get_routes_url(router_name).as_str(), + get_routes_url(vpc_name, router_name).as_str(), ) .await .items; @@ -125,12 +144,12 @@ async fn test_router_routes_crud_operations( assert_eq!(routes.len(), 0); let route_name = "custom-route"; - let route_url = get_route_url(router_name, route_name); + let route_url = get_route_url(vpc_name, router_name, route_name); // Create a new custom route let route_created: RouterRoute = NexusRequest::objects_post( client, - get_routes_url(router_name).as_str(), + get_routes_url(vpc_name, router_name).as_str(), ¶ms::RouterRouteCreate { identity: IdentityMetadataCreateParams { name: route_name.parse().unwrap(), @@ -212,7 +231,7 @@ async fn test_router_routes_crud_operations( client, StatusCode::NOT_FOUND, Method::GET, - get_route_url(router_name, route_name).as_str(), + get_route_url(vpc_name, router_name, route_name).as_str(), ) .authn_as(AuthnMode::PrivilegedUser) .execute() @@ -301,3 +320,219 @@ async fn test_router_routes_disallow_mixed_v4_v6( } } } + +#[nexus_test] +async fn test_router_routes_modify_system_routes( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + let _ = create_project(&client, PROJECT_NAME).await; + + let vpc_name = "default"; + + // Attempting to add a new route to a system router should fail. + let err = create_route_with_error( + client, + PROJECT_NAME, + vpc_name, + "system", + "bad-route", + "ipnet:240.0.0.0/8".parse().unwrap(), + "inetgw:outbound".parse().unwrap(), + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + err.message, + "user-provided routes cannot be added to a system router" + ); + + // Get the system router's routes + let [v4_route, v6_route, subnet_route] = + get_system_routes(client, vpc_name).await; + + // Attempting to modify a VPC subnet route should fail. + // Deletes are tested above. + let err = object_put_error( + client, + &get_route_url(vpc_name, "system", subnet_route.name().as_str()) + .as_str(), + &RouterRouteUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: None, + }, + target: "drop".parse().unwrap(), + destination: "subnet:default".parse().unwrap(), + }, + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + err.message, + "routes of type VpcSubnet within the system router are not modifiable" + ); + + // Modifying the target of a Default (gateway) route should succeed. + let v4_route: RouterRoute = object_put( + client, + &get_route_url(vpc_name, "system", v4_route.name().as_str()).as_str(), + &RouterRouteUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: None, + }, + destination: v4_route.destination, + target: "drop".parse().unwrap(), + }, + ) + .await; + assert_eq!(v4_route.target, RouteTarget::Drop); + + let v6_route: RouterRoute = object_put( + client, + &get_route_url(vpc_name, "system", v6_route.name().as_str()).as_str(), + &RouterRouteUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: None, + }, + destination: v6_route.destination, + target: "drop".parse().unwrap(), + }, + ) + .await; + assert_eq!(v6_route.target, RouteTarget::Drop); + + // Modifying the *destination* should not. + let err = object_put_error( + client, + &get_route_url(vpc_name, "system", v4_route.name().as_str()).as_str(), + &RouterRouteUpdate { + identity: IdentityMetadataUpdateParams { + name: None, + description: None, + }, + destination: "ipnet:10.0.0.0/8".parse().unwrap(), + target: "drop".parse().unwrap(), + }, + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + err.message, + "the destination and metadata of a Default route cannot be changed", + ); +} + +#[nexus_test] +async fn test_router_routes_internet_gateway_target( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + let _ = create_project(&client, PROJECT_NAME).await; + let vpc_name = "default"; + let router_name = "routy"; + let _router = + create_router(&client, PROJECT_NAME, vpc_name, router_name).await; + + // Internet gateways are not fully supported: only 'inetgw:outbound' + // is a valid choice. + let dest: RouteDestination = "ipnet:240.0.0.0/8".parse().unwrap(); + + let err = create_route_with_error( + client, + PROJECT_NAME, + vpc_name, + &router_name, + "bad-route", + dest.clone(), + "inetgw:not-a-real-gw".parse().unwrap(), + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + err.message, + "'outbound' is currently the only valid internet gateway" + ); + + // This can be used in a custom router, in addition + // to its default system spot. + let target: RouteTarget = "inetgw:outbound".parse().unwrap(); + let route = create_route( + client, + PROJECT_NAME, + vpc_name, + router_name, + "good-route", + dest.clone(), + target.clone(), + ) + .await; + assert_eq!(route.destination, dest); + assert_eq!(route.target, target); +} + +#[nexus_test] +async fn test_router_routes_disallow_custom_targets( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + let _ = create_project(&client, PROJECT_NAME).await; + let vpc_name = "default"; + let router_name = "routy"; + let _router = + create_router(&client, PROJECT_NAME, vpc_name, router_name).await; + + // Neither 'vpc:xxx' nor 'subnet:xxx' can be specified as route targets + // in custom routers. + let dest: RouteDestination = "ipnet:240.0.0.0/8".parse().unwrap(); + + let err = create_route_with_error( + client, + PROJECT_NAME, + vpc_name, + &router_name, + "bad-route", + dest.clone(), + "vpc:a-vpc-name-unknown".parse().unwrap(), + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + err.message, + "VPCs cannot be used as a destination or target in custom routers" + ); + + let err = create_route_with_error( + client, + PROJECT_NAME, + vpc_name, + &router_name, + "bad-route", + "vpc:a-vpc-name-unknown".parse().unwrap(), + "drop".parse().unwrap(), + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + err.message, + "VPCs cannot be used as a destination or target in custom routers" + ); + + let err = create_route_with_error( + client, + PROJECT_NAME, + vpc_name, + &router_name, + "bad-route", + dest.clone(), + "subnet:a-vpc-name-unknown".parse().unwrap(), + StatusCode::BAD_REQUEST, + ) + .await; + assert_eq!( + err.message, + "subnets cannot be used as a target in custom routers" + ); +} diff --git a/nexus/tests/integration_tests/vpc_routers.rs b/nexus/tests/integration_tests/vpc_routers.rs index 8d6bd14bf7..d805d5acef 100644 --- a/nexus/tests/integration_tests/vpc_routers.rs +++ b/nexus/tests/integration_tests/vpc_routers.rs @@ -25,6 +25,7 @@ use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::IdentityMetadataUpdateParams; use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::NameOrId; +use omicron_common::api::external::SimpleIdentity; pub const PROJECT_NAME: &str = "os-cartographers"; @@ -50,6 +51,22 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { assert_eq!(routers.len(), 1); assert_eq!(routers[0].kind, VpcRouterKind::System); + // This router should not be deletable. + let system_router_url = format!("/v1/vpc-routers/{}", routers[0].id()); + let error: dropshot::HttpErrorResponseBody = NexusRequest::expect_failure( + client, + StatusCode::BAD_REQUEST, + Method::DELETE, + &system_router_url, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + assert_eq!(error.message, "cannot delete system router"); + let router_name = "router1"; let router_url = format!( "/v1/vpc-routers/{}?project={}&vpc={}", @@ -414,7 +431,7 @@ async fn test_vpc_routers_attach_to_subnet( } #[nexus_test] -async fn test_vpc_routers_custom_route_at_instance( +async fn test_vpc_routers_custom_delivered_to_instance( cptestctx: &ControlPlaneTestContext, ) { let _client = &cptestctx.external_client; @@ -429,49 +446,6 @@ async fn test_vpc_routers_custom_route_at_instance( todo!() } -#[nexus_test] -async fn test_vpc_routers_modify_system_routes( - cptestctx: &ControlPlaneTestContext, -) { - let _client = &cptestctx.external_client; - - // Attempting to delete a system router should fail. - - // Attempting to add a new route to a system router should fail. - - // Attempting to modify/delete a VPC subnet route should fail. - - // Modifying the target of a Default (gateway) route should succeed. - - todo!() -} - -#[nexus_test] -async fn test_vpc_routers_internet_gateway_target( - cptestctx: &ControlPlaneTestContext, -) { - let _client = &cptestctx.external_client; - - // Internet gateways are not fully supported: only 'inetgw:outbound' - // is a valid choice. - - // This can be used in both system and custom routers. - - todo!() -} - -#[nexus_test] -async fn test_vpc_routers_disallow_custom_targets( - cptestctx: &ControlPlaneTestContext, -) { - let _client = &cptestctx.external_client; - - // Neither 'vpc:xxx' nor 'subnet:xxx' can be specified as route targets - // in custom routers. - - todo!() -} - async fn set_custom_router( client: &ClientTestContext, subnet_name: &str, diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index a003e2204b..c1123a3f96 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -1228,7 +1228,8 @@ pub struct VpcSubnetCreate { /// to any destination address. /// /// Custom routers apply in addition to the VPC-wide *system* router, and have - /// higher priority than + /// higher priority than the system router for an otherwise + /// equal-prefix-length match. pub custom_router: Option, } @@ -1240,9 +1241,6 @@ pub struct VpcSubnetUpdate { /// An optional router, used to direct packets sent from hosts in this subnet /// to any destination address. - /// - /// Custom routers apply in addition to the VPC-wide *system* router, and have - /// higher priority than pub custom_router: Option, } diff --git a/openapi/nexus.json b/openapi/nexus.json index a312db663f..2213052ffb 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -14817,7 +14817,7 @@ "type": "boolean" }, "transit_ips": { - "description": "TODO: describe", + "description": "A set of additional networks that this interface may send and receive traffic on.", "type": "array", "items": { "$ref": "#/components/schemas/IpNet" @@ -20172,7 +20172,7 @@ "properties": { "custom_router": { "nullable": true, - "description": "An optional router, used to direct packets sent from hosts in this subnet to any destination address.\n\nCustom routers apply in addition to the VPC-wide *system* router, and have higher priority than", + "description": "An optional router, used to direct packets sent from hosts in this subnet to any destination address.\n\nCustom routers apply in addition to the VPC-wide *system* router, and have higher priority than the system router for an otherwise equal-prefix-length match.", "allOf": [ { "$ref": "#/components/schemas/NameOrId" @@ -20236,7 +20236,7 @@ "properties": { "custom_router": { "nullable": true, - "description": "XXX", + "description": "An optional router, used to direct packets sent from hosts in this subnet to any destination address.", "allOf": [ { "$ref": "#/components/schemas/NameOrId" From 4c40916da43dba3ef55b23966ab4ed62bfcf9a0f Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 29 May 2024 23:51:01 +0100 Subject: [PATCH 38/59] Don't break parsing NICs from old blueprints --- common/src/api/internal/shared.rs | 2 +- illumos-utils/src/opte/port_manager.rs | 20 ++++++++++--------- nexus/db-model/src/network_interface.rs | 2 +- nexus/db-model/src/omicron_zone_config.rs | 2 +- .../src/db/datastore/network_interface.rs | 2 +- nexus/db-queries/src/db/datastore/rack.rs | 18 ++++++++--------- .../execution/src/external_networking.rs | 6 +++--- .../planning/src/blueprint_builder/builder.rs | 2 +- .../output/planner_nonprovisionable_2_2a.txt | 1 + nexus/test-utils/src/lib.rs | 4 ++-- openapi/nexus-internal.json | 2 +- openapi/nexus.json | 2 +- openapi/sled-agent.json | 2 +- schema/all-zone-requests.json | 9 +++++++++ schema/rss-service-plan-v3.json | 9 +++++++++ sled-agent/src/rack_setup/plan/service.rs | 6 +++--- sled-agent/src/sim/server.rs | 4 ++-- .../new-zones-ledgers/rack2-sled11.json | 3 ++- .../new-zones-ledgers/rack2-sled12.json | 6 ++++-- .../new-zones-ledgers/rack2-sled21.json | 6 ++++-- .../new-zones-ledgers/rack2-sled25.json | 3 ++- .../output/new-zones-ledgers/rack2-sled8.json | 3 ++- .../new-zones-ledgers/rack3-sled11.json | 3 ++- .../new-zones-ledgers/rack3-sled13.json | 3 ++- .../new-zones-ledgers/rack3-sled14.json | 3 ++- .../new-zones-ledgers/rack3-sled15.json | 3 ++- .../new-zones-ledgers/rack3-sled20.json | 3 ++- .../new-zones-ledgers/rack3-sled25.json | 3 ++- .../output/new-zones-ledgers/rack3-sled8.json | 3 ++- 29 files changed, 85 insertions(+), 50 deletions(-) diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 0215d5a9d0..84f01c2d8d 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -57,7 +57,7 @@ pub struct NetworkInterface { pub vni: external::Vni, pub primary: bool, pub slot: u8, - pub transit_ips: Vec, + pub transit_ips: Option>, } /// An IP address and port range used for source NAT, i.e., making diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 353d25858a..ec02104715 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -444,16 +444,18 @@ impl PortManager { // TODO: Currently set only in initial state. // This, external IPs, and cfg'able state // (DHCP?) are probably worth being managed by an RPW. - for block in &nic.transit_ips { - #[cfg(target_os = "illumos")] - hdl.allow_cidr(&port_name, super::net_to_cidr(*block)); + if let Some(blocks) = &nic.transit_ips { + for block in blocks { + #[cfg(target_os = "illumos")] + hdl.allow_cidr(&port_name, super::net_to_cidr(*block)); - debug!( - self.inner.log, - "Added CIDR to in/out allowlist"; - "port_name" => &port_name, - "cidr" => ?block, - ); + debug!( + self.inner.log, + "Added CIDR to in/out allowlist"; + "port_name" => &port_name, + "cidr" => ?block, + ); + } } info!( diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs index 5409ad66b3..8561d8f2c2 100644 --- a/nexus/db-model/src/network_interface.rs +++ b/nexus/db-model/src/network_interface.rs @@ -104,7 +104,7 @@ impl NetworkInterface { vni: external::Vni::try_from(0).unwrap(), primary: self.primary, slot: *self.slot, - transit_ips: self.transit_ips.into_iter().map(Into::into).collect(), + transit_ips: Some(self.transit_ips.into_iter().map(Into::into).collect()), } } } diff --git a/nexus/db-model/src/omicron_zone_config.rs b/nexus/db-model/src/omicron_zone_config.rs index 3b18a749a7..7f269c55e3 100644 --- a/nexus/db-model/src/omicron_zone_config.rs +++ b/nexus/db-model/src/omicron_zone_config.rs @@ -659,7 +659,7 @@ impl OmicronZoneNic { vni: omicron_common::api::external::Vni::try_from(*self.vni) .context("parsing VNI")?, subnet: self.subnet.into(), - transit_ips: vec![], + transit_ips: None, }) } } diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 79389bc232..ddf340749c 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -94,7 +94,7 @@ impl From for omicron_common::api::internal::shared::NetworkInterface { vni: nic.vni.0, primary: nic.primary, slot: u8::try_from(nic.slot).unwrap(), - transit_ips: nic.transit_ips.iter().map(|v| (*v).into()).collect(), + transit_ips: Some(nic.transit_ips.iter().map(|v| (*v).into()).collect()), } } } diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 5c7707777c..4c5e942f16 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1388,7 +1388,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, }, ), @@ -1418,7 +1418,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), @@ -1466,7 +1466,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, }, ), @@ -1496,7 +1496,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), @@ -1722,7 +1722,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, }, ), @@ -1755,7 +1755,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, }, ), @@ -1994,7 +1994,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, }, ), @@ -2100,7 +2100,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, }, ), @@ -2133,7 +2133,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, }, ), diff --git a/nexus/reconfigurator/execution/src/external_networking.rs b/nexus/reconfigurator/execution/src/external_networking.rs index 6e23879cb0..9794000d6b 100644 --- a/nexus/reconfigurator/execution/src/external_networking.rs +++ b/nexus/reconfigurator/execution/src/external_networking.rs @@ -500,7 +500,7 @@ mod tests { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }; let dns_id = OmicronZoneUuid::new_v4(); @@ -527,7 +527,7 @@ mod tests { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }; // Boundary NTP: @@ -557,7 +557,7 @@ mod tests { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }; Self { diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index c92925e3d6..3641bcb842 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -709,7 +709,7 @@ impl<'a> BlueprintBuilder<'a> { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, } }; diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt index 648c082c0f..eabc76b4e9 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt @@ -208,6 +208,7 @@ ERRORS: ), primary: true, slot: 0, + transit_ips: None, }, external_tls: false, external_dns_servers: [], diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index b70e512db8..9a7d6ba5b0 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -687,7 +687,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { slot: 0, subnet: (*NEXUS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, - transit_ips: vec![], + transit_ips: None, }, }), }); @@ -1044,7 +1044,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { slot: 0, subnet: (*DNS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, - transit_ips: vec![], + transit_ips: None, }, }, ), diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index f6fa1a48d9..d5fefdc348 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -3471,6 +3471,7 @@ "$ref": "#/components/schemas/IpNet" }, "transit_ips": { + "nullable": true, "type": "array", "items": { "$ref": "#/components/schemas/IpNet" @@ -3489,7 +3490,6 @@ "primary", "slot", "subnet", - "transit_ips", "vni" ] }, diff --git a/openapi/nexus.json b/openapi/nexus.json index 2213052ffb..13c7167db9 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -15785,6 +15785,7 @@ "$ref": "#/components/schemas/IpNet" }, "transit_ips": { + "nullable": true, "type": "array", "items": { "$ref": "#/components/schemas/IpNet" @@ -15803,7 +15804,6 @@ "primary", "slot", "subnet", - "transit_ips", "vni" ] }, diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 9673e2b239..b18cda6f19 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -3595,6 +3595,7 @@ "$ref": "#/components/schemas/IpNet" }, "transit_ips": { + "nullable": true, "type": "array", "items": { "$ref": "#/components/schemas/IpNet" @@ -3613,7 +3614,6 @@ "primary", "slot", "subnet", - "transit_ips", "vni" ] }, diff --git a/schema/all-zone-requests.json b/schema/all-zone-requests.json index 7fe9b139eb..3dbc3ef563 100644 --- a/schema/all-zone-requests.json +++ b/schema/all-zone-requests.json @@ -255,6 +255,15 @@ "subnet": { "$ref": "#/definitions/IpNet" }, + "transit_ips": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/IpNet" + } + }, "vni": { "$ref": "#/definitions/Vni" } diff --git a/schema/rss-service-plan-v3.json b/schema/rss-service-plan-v3.json index bab3e916ba..c26c5f29e6 100644 --- a/schema/rss-service-plan-v3.json +++ b/schema/rss-service-plan-v3.json @@ -253,6 +253,15 @@ "subnet": { "$ref": "#/definitions/IpNet" }, + "transit_ips": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/IpNet" + } + }, "vni": { "$ref": "#/definitions/Vni" } diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 2af0556d63..8bb890f841 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -1055,7 +1055,7 @@ impl ServicePortBuilder { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }; Some((nic, external_ip)) @@ -1096,7 +1096,7 @@ impl ServicePortBuilder { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }; Ok((nic, external_ip)) @@ -1154,7 +1154,7 @@ impl ServicePortBuilder { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }; Ok((nic, snat_cfg)) diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index dcbfb8b80e..bcf12ab3bf 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -409,7 +409,7 @@ pub async fn run_standalone_server( vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, external_tls: false, external_dns_servers: vec![], @@ -453,7 +453,7 @@ pub async fn run_standalone_server( vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: vec![], + transit_ips: None, }, }, }); diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled11.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled11.json index 79aae3e8c1..b412cfe391 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled11.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled11.json @@ -109,7 +109,8 @@ "subnet": "172.30.1.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null } } }, diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled12.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled12.json index 39ebad3183..550c040001 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled12.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled12.json @@ -148,7 +148,8 @@ "subnet": "172.30.2.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "external_tls": true, "external_dns_servers": [ @@ -200,7 +201,8 @@ "subnet": "172.30.3.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "snat_cfg": { "ip": "172.20.26.6", diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled21.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled21.json index 35caa638e8..ddccae6943 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled21.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled21.json @@ -50,7 +50,8 @@ "subnet": "172.30.2.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "external_tls": true, "external_dns_servers": [ @@ -200,7 +201,8 @@ "subnet": "172.30.3.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "snat_cfg": { "ip": "172.20.26.7", diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled25.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled25.json index 09a07149cf..ccfcdb7857 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled25.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled25.json @@ -137,7 +137,8 @@ "subnet": "172.30.1.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null } } }, diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled8.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled8.json index 669889b3c5..6f6c3cd2eb 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled8.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled8.json @@ -50,7 +50,8 @@ "subnet": "172.30.2.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "external_tls": true, "external_dns_servers": [ diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled11.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled11.json index c94417ffb8..79de32e47e 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled11.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled11.json @@ -169,7 +169,8 @@ "subnet": "172.30.3.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "snat_cfg": { "ip": "45.154.216.39", diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled13.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled13.json index 66c04be148..f518292390 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled13.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled13.json @@ -169,7 +169,8 @@ "subnet": "172.30.3.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "snat_cfg": { "ip": "45.154.216.38", diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled14.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled14.json index e8d061dbfd..b2e8a7a68c 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled14.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled14.json @@ -64,7 +64,8 @@ "subnet": "172.30.2.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "external_tls": true, "external_dns_servers": [ diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled15.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled15.json index e3b3dba86a..776827fd38 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled15.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled15.json @@ -25,7 +25,8 @@ "subnet": "172.30.1.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null } } }, diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled20.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled20.json index f02f1f05e5..55b7a99d30 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled20.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled20.json @@ -134,7 +134,8 @@ "subnet": "172.30.2.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "external_tls": true, "external_dns_servers": [ diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled25.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled25.json index 8deca6b56a..0efbf3693e 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled25.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled25.json @@ -81,7 +81,8 @@ "subnet": "172.30.1.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null } } }, diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled8.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled8.json index b848826231..1e4eebaf6f 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled8.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled8.json @@ -64,7 +64,8 @@ "subnet": "172.30.2.0/24", "vni": 100, "primary": true, - "slot": 0 + "slot": 0, + "transit_ips": null }, "external_tls": true, "external_dns_servers": [ From 536b40c2d2f59d446466bf167479821f9d593531 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 29 May 2024 23:57:23 +0100 Subject: [PATCH 39/59] Minor fmt + lint tweaks. --- illumos-utils/src/opte/port_manager.rs | 2 +- nexus/db-model/src/network_interface.rs | 4 +++- nexus/db-queries/src/db/datastore/network_interface.rs | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index ec02104715..a054f93de1 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -447,7 +447,7 @@ impl PortManager { if let Some(blocks) = &nic.transit_ips { for block in blocks { #[cfg(target_os = "illumos")] - hdl.allow_cidr(&port_name, super::net_to_cidr(*block)); + hdl.allow_cidr(&port_name, super::net_to_cidr(*block))?; debug!( self.inner.log, diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs index 8561d8f2c2..658361c7b0 100644 --- a/nexus/db-model/src/network_interface.rs +++ b/nexus/db-model/src/network_interface.rs @@ -104,7 +104,9 @@ impl NetworkInterface { vni: external::Vni::try_from(0).unwrap(), primary: self.primary, slot: *self.slot, - transit_ips: Some(self.transit_ips.into_iter().map(Into::into).collect()), + transit_ips: Some( + self.transit_ips.into_iter().map(Into::into).collect(), + ), } } } diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index ddf340749c..a83fe2dcb5 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -94,7 +94,9 @@ impl From for omicron_common::api::internal::shared::NetworkInterface { vni: nic.vni.0, primary: nic.primary, slot: u8::try_from(nic.slot).unwrap(), - transit_ips: Some(nic.transit_ips.iter().map(|v| (*v).into()).collect()), + transit_ips: Some( + nic.transit_ips.iter().map(|v| (*v).into()).collect(), + ), } } } From 83b171091690a809d2cd7c612d25bbe78a1d1680 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 30 May 2024 00:50:22 +0100 Subject: [PATCH 40/59] Fixup unexpected authz endpoints. --- nexus/tests/output/unexpected-authz-endpoints.txt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/nexus/tests/output/unexpected-authz-endpoints.txt b/nexus/tests/output/unexpected-authz-endpoints.txt index e8bb60224a..cd05058762 100644 --- a/nexus/tests/output/unexpected-authz-endpoints.txt +++ b/nexus/tests/output/unexpected-authz-endpoints.txt @@ -1,13 +1,3 @@ API endpoints tested by unauthorized.rs but not found in the OpenAPI spec: -GET "/v1/vpc-routers?project=demo-project&vpc=demo-vpc" -POST "/v1/vpc-routers?project=demo-project&vpc=demo-vpc" -GET "/v1/vpc-routers/demo-vpc-router?project=demo-project&vpc=demo-vpc" -PUT "/v1/vpc-routers/demo-vpc-router?project=demo-project&vpc=demo-vpc" -DELETE "/v1/vpc-routers/demo-vpc-router?project=demo-project&vpc=demo-vpc" -GET "/v1/vpc-router-routes?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" -POST "/v1/vpc-router-routes?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" -GET "/v1/vpc-router-routes/demo-router-route?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" -PUT "/v1/vpc-router-routes/demo-router-route?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" -DELETE "/v1/vpc-router-routes/demo-router-route?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" PUT "/v1/system/update/repository?file_name=demo-repo.zip" GET "/v1/system/update/repository/1.0.0" From 7413adfc738c7c8fa8bb70d3c49a244bb2563885 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 30 May 2024 11:41:16 +0100 Subject: [PATCH 41/59] Last big test, cutesier test data names. --- nexus/tests/integration_tests/instances.rs | 4 +- .../tests/integration_tests/router_routes.rs | 49 +-- nexus/tests/integration_tests/vpc_routers.rs | 313 +++++++++++++++--- 3 files changed, 295 insertions(+), 71 deletions(-) diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 41e2588f7c..2d71a00155 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -4731,7 +4731,7 @@ pub async fn assert_sled_vpc_routes( datastore: &DataStore, subnet_id: Uuid, vni: Vni, -) { +) -> (HashSet, HashSet) { let (.., authz_vpc, _, db_subnet) = LookupPath::new(opctx, datastore) .vpc_subnet_id(subnet_id) .fetch() @@ -4790,6 +4790,8 @@ pub async fn assert_sled_vpc_routes( ) .await .expect("matching vpc routes should be present"); + + (system_routes, custom_routes) } /// Simulate completion of an ongoing instance state transition. To do this, we diff --git a/nexus/tests/integration_tests/router_routes.rs b/nexus/tests/integration_tests/router_routes.rs index a0aaa596c7..8af5c1b44a 100644 --- a/nexus/tests/integration_tests/router_routes.rs +++ b/nexus/tests/integration_tests/router_routes.rs @@ -31,6 +31,8 @@ use nexus_test_utils::resource_helpers::{ }; use crate::integration_tests::vpc_routers::PROJECT_NAME; +use crate::integration_tests::vpc_routers::ROUTER_NAMES; +use crate::integration_tests::vpc_routers::VPC_NAME; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -245,11 +247,11 @@ async fn test_router_routes_disallow_mixed_v4_v6( ) { let client = &cptestctx.external_client; let _ = create_project(&client, PROJECT_NAME).await; + let _ = create_vpc(&client, PROJECT_NAME, VPC_NAME).await; - let vpc_name = "default"; - let router_name = "routy"; + let router_name = ROUTER_NAMES[0]; let _router = - create_router(&client, PROJECT_NAME, vpc_name, router_name).await; + create_router(&client, PROJECT_NAME, VPC_NAME, router_name).await; // Some targets/strings refer to a mixed v4/v6 entity, e.g., // subnet or instance. Others refer to one kind only (ipnet, ip). @@ -294,7 +296,7 @@ async fn test_router_routes_disallow_mixed_v4_v6( create_route( client, PROJECT_NAME, - vpc_name, + VPC_NAME, router_name, &route_name, dest, @@ -305,7 +307,7 @@ async fn test_router_routes_disallow_mixed_v4_v6( let err = create_route_with_error( client, PROJECT_NAME, - vpc_name, + VPC_NAME, router_name, &route_name, dest, @@ -327,14 +329,13 @@ async fn test_router_routes_modify_system_routes( ) { let client = &cptestctx.external_client; let _ = create_project(&client, PROJECT_NAME).await; - - let vpc_name = "default"; + let _ = create_vpc(&client, PROJECT_NAME, VPC_NAME).await; // Attempting to add a new route to a system router should fail. let err = create_route_with_error( client, PROJECT_NAME, - vpc_name, + VPC_NAME, "system", "bad-route", "ipnet:240.0.0.0/8".parse().unwrap(), @@ -349,13 +350,13 @@ async fn test_router_routes_modify_system_routes( // Get the system router's routes let [v4_route, v6_route, subnet_route] = - get_system_routes(client, vpc_name).await; + get_system_routes(client, VPC_NAME).await; // Attempting to modify a VPC subnet route should fail. // Deletes are tested above. let err = object_put_error( client, - &get_route_url(vpc_name, "system", subnet_route.name().as_str()) + &get_route_url(VPC_NAME, "system", subnet_route.name().as_str()) .as_str(), &RouterRouteUpdate { identity: IdentityMetadataUpdateParams { @@ -376,7 +377,7 @@ async fn test_router_routes_modify_system_routes( // Modifying the target of a Default (gateway) route should succeed. let v4_route: RouterRoute = object_put( client, - &get_route_url(vpc_name, "system", v4_route.name().as_str()).as_str(), + &get_route_url(VPC_NAME, "system", v4_route.name().as_str()).as_str(), &RouterRouteUpdate { identity: IdentityMetadataUpdateParams { name: None, @@ -391,7 +392,7 @@ async fn test_router_routes_modify_system_routes( let v6_route: RouterRoute = object_put( client, - &get_route_url(vpc_name, "system", v6_route.name().as_str()).as_str(), + &get_route_url(VPC_NAME, "system", v6_route.name().as_str()).as_str(), &RouterRouteUpdate { identity: IdentityMetadataUpdateParams { name: None, @@ -407,7 +408,7 @@ async fn test_router_routes_modify_system_routes( // Modifying the *destination* should not. let err = object_put_error( client, - &get_route_url(vpc_name, "system", v4_route.name().as_str()).as_str(), + &get_route_url(VPC_NAME, "system", v4_route.name().as_str()).as_str(), &RouterRouteUpdate { identity: IdentityMetadataUpdateParams { name: None, @@ -431,10 +432,10 @@ async fn test_router_routes_internet_gateway_target( ) { let client = &cptestctx.external_client; let _ = create_project(&client, PROJECT_NAME).await; - let vpc_name = "default"; - let router_name = "routy"; + let _ = create_vpc(&client, PROJECT_NAME, VPC_NAME).await; + let router_name = ROUTER_NAMES[0]; let _router = - create_router(&client, PROJECT_NAME, vpc_name, router_name).await; + create_router(&client, PROJECT_NAME, VPC_NAME, router_name).await; // Internet gateways are not fully supported: only 'inetgw:outbound' // is a valid choice. @@ -443,7 +444,7 @@ async fn test_router_routes_internet_gateway_target( let err = create_route_with_error( client, PROJECT_NAME, - vpc_name, + VPC_NAME, &router_name, "bad-route", dest.clone(), @@ -462,7 +463,7 @@ async fn test_router_routes_internet_gateway_target( let route = create_route( client, PROJECT_NAME, - vpc_name, + VPC_NAME, router_name, "good-route", dest.clone(), @@ -479,10 +480,10 @@ async fn test_router_routes_disallow_custom_targets( ) { let client = &cptestctx.external_client; let _ = create_project(&client, PROJECT_NAME).await; - let vpc_name = "default"; - let router_name = "routy"; + let _ = create_vpc(&client, PROJECT_NAME, VPC_NAME).await; + let router_name = ROUTER_NAMES[0]; let _router = - create_router(&client, PROJECT_NAME, vpc_name, router_name).await; + create_router(&client, PROJECT_NAME, VPC_NAME, router_name).await; // Neither 'vpc:xxx' nor 'subnet:xxx' can be specified as route targets // in custom routers. @@ -491,7 +492,7 @@ async fn test_router_routes_disallow_custom_targets( let err = create_route_with_error( client, PROJECT_NAME, - vpc_name, + VPC_NAME, &router_name, "bad-route", dest.clone(), @@ -507,7 +508,7 @@ async fn test_router_routes_disallow_custom_targets( let err = create_route_with_error( client, PROJECT_NAME, - vpc_name, + VPC_NAME, &router_name, "bad-route", "vpc:a-vpc-name-unknown".parse().unwrap(), @@ -523,7 +524,7 @@ async fn test_router_routes_disallow_custom_targets( let err = create_route_with_error( client, PROJECT_NAME, - vpc_name, + VPC_NAME, &router_name, "bad-route", dest.clone(), diff --git a/nexus/tests/integration_tests/vpc_routers.rs b/nexus/tests/integration_tests/vpc_routers.rs index d805d5acef..9b56d8f055 100644 --- a/nexus/tests/integration_tests/vpc_routers.rs +++ b/nexus/tests/integration_tests/vpc_routers.rs @@ -2,13 +2,20 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::integration_tests::instances::assert_sled_vpc_routes; +use crate::integration_tests::instances::instance_simulate; use dropshot::test_util::ClientTestContext; use http::method::Method; use http::StatusCode; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::lookup::LookupPath; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::identity_eq; +use nexus_test_utils::resource_helpers::create_default_ip_pool; +use nexus_test_utils::resource_helpers::create_instance_with; +use nexus_test_utils::resource_helpers::create_route; use nexus_test_utils::resource_helpers::create_router; use nexus_test_utils::resource_helpers::create_vpc_subnet; use nexus_test_utils::resource_helpers::object_delete; @@ -17,6 +24,8 @@ use nexus_test_utils::resource_helpers::{create_project, create_vpc}; use nexus_test_utils::resource_helpers::{object_put, object_put_error}; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; +use nexus_types::external_api::params::InstanceNetworkInterfaceAttachment; +use nexus_types::external_api::params::InstanceNetworkInterfaceCreate; use nexus_types::external_api::params::VpcSubnetUpdate; use nexus_types::external_api::views::VpcRouter; use nexus_types::external_api::views::VpcRouterKind; @@ -26,8 +35,15 @@ use omicron_common::api::external::IdentityMetadataUpdateParams; use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::NameOrId; use omicron_common::api::external::SimpleIdentity; +use omicron_common::api::internal::shared::ResolvedVpcRoute; +use omicron_common::api::internal::shared::RouterTarget; +use std::collections::HashMap; -pub const PROJECT_NAME: &str = "os-cartographers"; +pub const PROJECT_NAME: &str = "cartographer"; +pub const VPC_NAME: &str = "the-isles"; +pub const SUBNET_NAMES: &[&str] = &["scotia", "albion", "eire"]; +const INSTANCE_NAMES: &[&str] = &["glaschu", "londinium"]; +pub const ROUTER_NAMES: &[&str] = &["cycle-network", "motorways"]; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -40,14 +56,13 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { let _ = create_project(&client, PROJECT_NAME).await; // Create a VPC. - let vpc_name = "vpc1"; - let vpc = create_vpc(&client, PROJECT_NAME, vpc_name).await; + let vpc = create_vpc(&client, PROJECT_NAME, VPC_NAME).await; let routers_url = - format!("/v1/vpc-routers?project={}&vpc={}", PROJECT_NAME, vpc_name); + format!("/v1/vpc-routers?project={}&vpc={}", PROJECT_NAME, VPC_NAME); // get routers should have only the system router created w/ the VPC - let routers = list_routers(client, &vpc_name).await; + let routers = list_routers(client, &VPC_NAME).await; assert_eq!(routers.len(), 1); assert_eq!(routers[0].kind, VpcRouterKind::System); @@ -67,10 +82,10 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { .unwrap(); assert_eq!(error.message, "cannot delete system router"); - let router_name = "router1"; + let router_name = ROUTER_NAMES[0]; let router_url = format!( "/v1/vpc-routers/{}?project={}&vpc={}", - router_name, PROJECT_NAME, vpc_name + router_name, PROJECT_NAME, VPC_NAME ); // fetching a particular router should 404 @@ -86,11 +101,14 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { .unwrap() .parsed_body() .unwrap(); - assert_eq!(error.message, "not found: vpc-router with name \"router1\""); + assert_eq!( + error.message, + format!("not found: vpc-router with name \"{router_name}\"") + ); // Create a VPC Router. let router = - create_router(&client, PROJECT_NAME, vpc_name, router_name).await; + create_router(&client, PROJECT_NAME, VPC_NAME, router_name).await; assert_eq!(router.identity.name, router_name); assert_eq!(router.identity.description, "router description"); assert_eq!(router.vpc_id, vpc.identity.id); @@ -107,7 +125,7 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { routers_eq(&router, &same_router); // routers list should now have the one in it - let routers = list_routers(client, &vpc_name).await; + let routers = list_routers(client, &VPC_NAME).await; assert_eq!(routers.len(), 2); routers_eq(&routers[0], &router); @@ -128,12 +146,15 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { .unwrap() .parsed_body() .unwrap(); - assert_eq!(error.message, "already exists: vpc-router \"router1\""); + assert_eq!( + error.message, + format!("already exists: vpc-router \"{router_name}\"") + ); - let router2_name = "router2"; + let router2_name = ROUTER_NAMES[1]; let router2_url = format!( "/v1/vpc-routers/{}?project={}&vpc={}", - router2_name, PROJECT_NAME, vpc_name + router2_name, PROJECT_NAME, VPC_NAME ); // second router 404s before it's created @@ -149,17 +170,20 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { .unwrap() .parsed_body() .unwrap(); - assert_eq!(error.message, "not found: vpc-router with name \"router2\""); + assert_eq!( + error.message, + format!("not found: vpc-router with name \"{router2_name}\"") + ); // create second custom router let router2 = - create_router(client, PROJECT_NAME, vpc_name, router2_name).await; + create_router(client, PROJECT_NAME, VPC_NAME, router2_name).await; assert_eq!(router2.identity.name, router2_name); assert_eq!(router2.vpc_id, vpc.identity.id); assert_eq!(router2.kind, VpcRouterKind::Custom); // routers list should now have two custom and one system - let routers = list_routers(client, &vpc_name).await; + let routers = list_routers(client, &VPC_NAME).await; assert_eq!(routers.len(), 3); routers_eq(&routers[0], &router); routers_eq(&routers[1], &router2); @@ -199,11 +223,14 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { .unwrap() .parsed_body() .unwrap(); - assert_eq!(error.message, "not found: vpc-router with name \"router1\""); + assert_eq!( + error.message, + format!("not found: vpc-router with name \"{router_name}\"") + ); let router_url = format!( "/v1/vpc-routers/new-name?project={}&vpc={}", - PROJECT_NAME, vpc_name + PROJECT_NAME, VPC_NAME ); // fetching by new name works @@ -215,13 +242,17 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { .unwrap() .parsed_body() .unwrap(); + routers_eq(&update, &updated_router); assert_eq!(&updated_router.identity.description, "another description"); // fetching list should show updated one - let routers = list_routers(client, &vpc_name).await; + let routers = list_routers(client, &VPC_NAME).await; assert_eq!(routers.len(), 3); - routers_eq(&routers[0], &updated_router); + routers_eq( + &routers.iter().find(|v| v.name().as_str() == "new-name").unwrap(), + &updated_router, + ); // delete first router NexusRequest::object_delete(&client, &router_url) @@ -231,7 +262,7 @@ async fn test_vpc_routers_crud_operations(cptestctx: &ControlPlaneTestContext) { .unwrap(); // routers list should now have two again, one system and one custom - let routers = list_routers(client, &vpc_name).await; + let routers = list_routers(client, &VPC_NAME).await; assert_eq!(routers.len(), 2); routers_eq(&routers[0], &router2); @@ -282,31 +313,30 @@ async fn test_vpc_routers_attach_to_subnet( let client = &cptestctx.external_client; // Create a project that we'll use for testing. - // This includes the vpc 'default'. let _ = create_project(&client, PROJECT_NAME).await; + let _ = create_vpc(&client, PROJECT_NAME, VPC_NAME).await; - let vpc_name = "default"; let subnet_name = "default"; let subnets_url = - format!("/v1/vpc-subnets?project={}&vpc={}", PROJECT_NAME, vpc_name); + format!("/v1/vpc-subnets?project={}&vpc={}", PROJECT_NAME, VPC_NAME); // get routers should have only the system router created w/ the VPC - let routers = list_routers(client, vpc_name).await; + let routers = list_routers(client, VPC_NAME).await; assert_eq!(routers.len(), 1); assert_eq!(routers[0].kind, VpcRouterKind::System); // Create a custom router for later use. - let router_name = "routy"; + let router_name = ROUTER_NAMES[0]; let router = - create_router(&client, PROJECT_NAME, vpc_name, router_name).await; + create_router(&client, PROJECT_NAME, VPC_NAME, router_name).await; assert_eq!(router.kind, VpcRouterKind::Custom); // Attaching a system router should fail. let err = object_put_error( client, &format!( - "/v1/vpc-subnets/{subnet_name}?project={PROJECT_NAME}&vpc={vpc_name}" + "/v1/vpc-subnets/{subnet_name}?project={PROJECT_NAME}&vpc={VPC_NAME}" ), &VpcSubnetUpdate { identity: IdentityMetadataUpdateParams { @@ -324,7 +354,7 @@ async fn test_vpc_routers_attach_to_subnet( let default_subnet = set_custom_router( client, "default", - vpc_name, + VPC_NAME, Some(router.identity.id.into()), ) .await; @@ -332,11 +362,11 @@ async fn test_vpc_routers_attach_to_subnet( // Attaching a custom router to another subnet (same VPC) should succeed: // ... at create time. - let subnet2_name = "subnetty"; + let subnet2_name = SUBNET_NAMES[0]; let subnet2 = create_vpc_subnet( &client, &PROJECT_NAME, - &vpc_name, + &VPC_NAME, &subnet2_name, Ipv4Net("192.168.0.0/24".parse().unwrap()), None, @@ -346,11 +376,11 @@ async fn test_vpc_routers_attach_to_subnet( assert_eq!(subnet2.custom_router_id, Some(router.identity.id)); // ... and via update. - let subnet3_name = "subnettier"; + let subnet3_name = SUBNET_NAMES[1]; let _ = create_vpc_subnet( &client, &PROJECT_NAME, - &vpc_name, + &VPC_NAME, &subnet3_name, Ipv4Net("192.168.1.0/24".parse().unwrap()), None, @@ -361,7 +391,7 @@ async fn test_vpc_routers_attach_to_subnet( let subnet3 = set_custom_router( client, subnet3_name, - vpc_name, + VPC_NAME, Some(router.identity.id.into()), ) .await; @@ -385,19 +415,19 @@ async fn test_vpc_routers_attach_to_subnet( assert_eq!(err.message, "router and subnet must belong to the same VPC"); // Detach (and double detach) should succeed without issue. - let subnet3 = set_custom_router(client, subnet3_name, vpc_name, None).await; + let subnet3 = set_custom_router(client, subnet3_name, VPC_NAME, None).await; assert_eq!(subnet3.custom_router_id, None); - let subnet3 = set_custom_router(client, subnet3_name, vpc_name, None).await; + let subnet3 = set_custom_router(client, subnet3_name, VPC_NAME, None).await; assert_eq!(subnet3.custom_router_id, None); // Assigning a new router should not require that we first detach the old one. - let router2_name = "routier"; + let router2_name = ROUTER_NAMES[1]; let router2 = - create_router(&client, PROJECT_NAME, vpc_name, router2_name).await; + create_router(&client, PROJECT_NAME, VPC_NAME, router2_name).await; let subnet2 = set_custom_router( client, subnet2_name, - vpc_name, + VPC_NAME, Some(router2.identity.id.into()), ) .await; @@ -407,7 +437,7 @@ async fn test_vpc_routers_attach_to_subnet( let subnet2 = set_custom_router( client, subnet2_name, - vpc_name, + VPC_NAME, Some(router.identity.id.into()), ) .await; @@ -417,8 +447,7 @@ async fn test_vpc_routers_attach_to_subnet( object_delete( &client, &format!( - "/v1/vpc-routers/{router_name}?vpc={}&project={PROJECT_NAME}", - "default" + "/v1/vpc-routers/{router_name}?vpc={VPC_NAME}&project={PROJECT_NAME}", ), ) .await; @@ -434,16 +463,208 @@ async fn test_vpc_routers_attach_to_subnet( async fn test_vpc_routers_custom_delivered_to_instance( cptestctx: &ControlPlaneTestContext, ) { - let _client = &cptestctx.external_client; + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.server_context(); + let nexus = &apictx.nexus; + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + // Create some instances, one per subnet, and a default pool etc. + create_default_ip_pool(client).await; + create_project(client, PROJECT_NAME).await; + + let vpc = create_vpc(&client, PROJECT_NAME, VPC_NAME).await; + + let mut subnets = vec![]; + let mut instances = vec![]; + let mut instance_nics = HashMap::new(); + for (i, (subnet_name, instance_name)) in + SUBNET_NAMES.iter().zip(INSTANCE_NAMES.iter()).enumerate() + { + let subnet = create_vpc_subnet( + &client, + PROJECT_NAME, + VPC_NAME, + subnet_name, + Ipv4Net(format!("192.168.{i}.0/24").parse().unwrap()), + None, + None, + ) + .await; + + let instance = create_instance_with( + client, + PROJECT_NAME, + instance_name, + &InstanceNetworkInterfaceAttachment::Create(vec![ + InstanceNetworkInterfaceCreate { + identity: IdentityMetadataCreateParams { + name: format!("nic-{i}").parse().unwrap(), + description: "".into(), + }, + vpc_name: vpc.name().clone(), + subnet_name: subnet_name.parse().unwrap(), + ip: Some(format!("192.168.{i}.10").parse().unwrap()), + }, + ]), + vec![], + vec![], + true, + ) + .await; + instance_simulate(nexus, &instance.identity.id).await; + + let (.., authz_instance) = LookupPath::new(&opctx, &datastore) + .instance_id(instance.identity.id) + .lookup_for(nexus_db_queries::authz::Action::Read) + .await + .unwrap(); + + let guest_nics = datastore + .derive_guest_network_interface_info(&opctx, &authz_instance) + .await + .unwrap(); + + instance_nics.insert(*instance_name, guest_nics); + subnets.push(subnet); + instances.push(instance); + } + + let sled_agent = &cptestctx.sled_agent.sled_agent; + + // Create some routers! + let mut routers = vec![]; + for router_name in ROUTER_NAMES { + let router = + create_router(&client, PROJECT_NAME, VPC_NAME, router_name).await; + + routers.push(router); + } + + let vni = instance_nics[INSTANCE_NAMES[0]][0].vni; // Installing a custom router onto a subnet with a live instance - // should install routes at that sled. + // should install routes at that sled. We should only have one sled. + // First, assert the default state. + for subnet in &subnets { + let (_system, custom) = assert_sled_vpc_routes( + &sled_agent, + &opctx, + &datastore, + subnet.id(), + vni, + ) + .await; + + assert!(custom.is_empty()); + } + + // Push a distinct route into each router and attach to each subnet. + for i in 0..2 { + create_route( + &client, + PROJECT_NAME, + VPC_NAME, + ROUTER_NAMES[i], + "a-sharp-drop", + format!("ipnet:24{i}.0.0.0/8").parse().unwrap(), + "drop".parse().unwrap(), + ) + .await; + + set_custom_router( + &client, + SUBNET_NAMES[i], + VPC_NAME, + Some(NameOrId::Name(ROUTER_NAMES[i].parse().unwrap())), + ) + .await; + } + + // Re-verify, assert that new routes are resolved correctly. + // Vec<(System, Custom)>. + let mut last_routes = vec![]; + for subnet in &subnets { + last_routes.push( + assert_sled_vpc_routes( + &sled_agent, + &opctx, + &datastore, + subnet.id(), + vni, + ) + .await, + ); + } + + assert!(last_routes[0].1.contains(&ResolvedVpcRoute { + dest: "240.0.0.0/8".parse().unwrap(), + target: RouterTarget::Drop + })); + assert!(last_routes[1].1.contains(&ResolvedVpcRoute { + dest: "241.0.0.0/8".parse().unwrap(), + target: RouterTarget::Drop + })); + + // Adding a new route should propagate that out to sleds. + create_route( + &client, + PROJECT_NAME, + VPC_NAME, + ROUTER_NAMES[0], + "ncn-74", + "ipnet:2.0.7.0/24".parse().unwrap(), + format!("instance:{}", INSTANCE_NAMES[1]).parse().unwrap(), + ) + .await; + + let (new_system, new_custom) = assert_sled_vpc_routes( + &sled_agent, + &opctx, + &datastore, + subnets[0].id(), + vni, + ) + .await; + + assert_eq!(last_routes[0].0, new_system); + assert!(new_custom.contains(&ResolvedVpcRoute { + dest: "2.0.7.0/24".parse().unwrap(), + target: RouterTarget::Ip(instance_nics[INSTANCE_NAMES[1]][0].ip) + })); // Swapping router should change the installed routes at that sled. + set_custom_router( + &client, + SUBNET_NAMES[0], + VPC_NAME, + Some(NameOrId::Name(ROUTER_NAMES[1].parse().unwrap())), + ) + .await; + let (new_system, new_custom) = assert_sled_vpc_routes( + &sled_agent, + &opctx, + &datastore, + subnets[0].id(), + vni, + ) + .await; + assert_eq!(last_routes[0].0, new_system); + assert_eq!(last_routes[1].1, new_custom); // Unsetting a router should remove affected non-system routes. - - todo!() + set_custom_router(&client, SUBNET_NAMES[0], VPC_NAME, None).await; + let (new_system, new_custom) = assert_sled_vpc_routes( + &sled_agent, + &opctx, + &datastore, + subnets[0].id(), + vni, + ) + .await; + assert_eq!(last_routes[0].0, new_system); + assert!(new_custom.is_empty()); } async fn set_custom_router( From bd7294c84a6b5e37d8afb6a90a55f97c9ca51fba Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 30 May 2024 17:46:30 +0100 Subject: [PATCH 42/59] Fixup one test. --- schema/all-zones-requests.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/schema/all-zones-requests.json b/schema/all-zones-requests.json index bb4dba2520..87bd570107 100644 --- a/schema/all-zones-requests.json +++ b/schema/all-zones-requests.json @@ -139,6 +139,15 @@ "subnet": { "$ref": "#/definitions/IpNet" }, + "transit_ips": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/IpNet" + } + }, "vni": { "$ref": "#/definitions/Vni" } From e3b7b90e86fb37ebc6bbd1a901f875e032a969e9 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 31 May 2024 13:36:52 +0100 Subject: [PATCH 43/59] Self-review. --- nexus/src/app/vpc_subnet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/src/app/vpc_subnet.rs b/nexus/src/app/vpc_subnet.rs index 05ce3ada87..7b323df49e 100644 --- a/nexus/src/app/vpc_subnet.rs +++ b/nexus/src/app/vpc_subnet.rs @@ -255,7 +255,7 @@ impl super::Nexus { let (.., authz_vpc, authz_subnet) = vpc_subnet_lookup.lookup_for(authz::Action::Modify).await?; - // Updating the subnet is a separate action. + // Updating the custom router is a separate action. self.vpc_subnet_update_custom_router( opctx, &authz_vpc, From cca764fea241ffe77f8c5aa6ba89874f6e17ad83 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 13 Jun 2024 14:02:36 +0100 Subject: [PATCH 44/59] Bump image. --- .github/buildomat/jobs/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 2dde4286dc..d15168d31f 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -2,7 +2,7 @@ #: #: name = "helios / deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.31" +#: target = "lab-2.0-opte-0.32" #: output_rules = [ #: "%/var/svc/log/oxide-sled-agent:default.log*", #: "%/zone/oxz_*/root/var/svc/log/oxide-*.log*", From bf3ffb02d6bb653b06c2bc3410b56e6b6db4743b Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 13 Jun 2024 14:12:01 +0100 Subject: [PATCH 45/59] Handle greater specificity of rule addition. --- illumos-utils/src/opte/port_manager.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index a054f93de1..f3775fc07c 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -447,7 +447,26 @@ impl PortManager { if let Some(blocks) = &nic.transit_ips { for block in blocks { #[cfg(target_os = "illumos")] - hdl.allow_cidr(&port_name, super::net_to_cidr(*block))?; + { + use oxide_vpc::api::Direction; + + // In principle if this were an operation on an existing + // port, we would explicitly undo the In addition if the + // Out addition fails. + // However, failure here will just destroy the port + // outright -- this should only happen if an excessive + // number of rules are specified. + hdl.allow_cidr( + &port_name, + super::net_to_cidr(*block), + Direction::In, + )?; + hdl.allow_cidr( + &port_name, + super::net_to_cidr(*block), + Direction::Out, + )?; + } debug!( self.inner.log, From 2425016f117891ae619619703bc6c3c94b63294c Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 13 Jun 2024 17:28:04 +0100 Subject: [PATCH 46/59] Forgot some maghemite SHAs... --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6c393a3c1..a8cbbc4307 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1591,7 +1591,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=c9824727eedc66d4920e42e7260df05050841ab8#c9824727eedc66d4920e42e7260df05050841ab8" +source = "git+https://github.com/oxidecomputer/maghemite?rev=e63f6d408908b3332d7cd89a4dd44a0f980d931d#e63f6d408908b3332d7cd89a4dd44a0f980d931d" dependencies = [ "percent-encoding", "progenitor", @@ -4275,7 +4275,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=c9824727eedc66d4920e42e7260df05050841ab8#c9824727eedc66d4920e42e7260df05050841ab8" +source = "git+https://github.com/oxidecomputer/maghemite?rev=e63f6d408908b3332d7cd89a4dd44a0f980d931d#e63f6d408908b3332d7cd89a4dd44a0f980d931d" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 1ba083ae47..0896ebc199 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -318,8 +318,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.12" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "c9824727eedc66d4920e42e7260df05050841ab8" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "c9824727eedc66d4920e42e7260df05050841ab8" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" } multimap = "0.10.0" nexus-auth = { path = "nexus/auth" } nexus-client = { path = "clients/nexus-client" } From 30e40437a0ad92133cadefe63cb6f7d0281d59a5 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 10:55:57 +0100 Subject: [PATCH 47/59] Minor fixes post-merge. --- nexus/db-queries/src/db/datastore/vpc.rs | 5 +++-- nexus/tests/integration_tests/instances.rs | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 6b33e2da8d..17e0c7ceb8 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1801,6 +1801,7 @@ mod tests { use omicron_common::api::external::Generation; use omicron_test_utils::dev; use omicron_uuid_kinds::GenericUuid; + use omicron_uuid_kinds::InstanceUuid; use omicron_uuid_kinds::SledUuid; use oxnet::IpNet; use oxnet::Ipv4Net; @@ -2666,7 +2667,7 @@ mod tests { &opctx, &authz_project, db::model::Instance::new( - Uuid::new_v4(), + InstanceUuid::new_v4(), authz_project.id(), ¶ms::InstanceCreate { identity: IdentityMetadataCreateParams { @@ -2711,7 +2712,7 @@ mod tests { &authz_instance, IncompleteNetworkInterface::new_instance( Uuid::new_v4(), - db_inst.id(), + InstanceUuid::from_untyped_uuid(db_inst.id()), sub0, IdentityMetadataCreateParams { name: "nic".parse().unwrap(), diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index ed9b14a3e4..f7f970c625 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -686,7 +686,8 @@ async fn test_instance_start_creates_networking_state( let mut checked = false; for agent in &sled_agents { - if Some(agent.id) == with_vmm.sled_id() { + if Some(agent.id) == with_vmm.sled_id().map(SledUuid::into_untyped_uuid) + { assert_sled_vpc_routes( agent, &opctx, From 0057228378738ed5d8bab888d861fa6c1c6ad994 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 11:15:34 +0100 Subject: [PATCH 48/59] Review feedback. --- nexus/db-queries/src/db/datastore/vpc.rs | 82 +++++++++++++----------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 17e0c7ceb8..a265853cb5 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -46,6 +46,7 @@ use chrono::Utc; use diesel::prelude::*; use diesel::result::DatabaseErrorKind; use diesel::result::Error as DieselError; +use futures::stream::{self, StreamExt}; use ipnetwork::IpNetwork; use nexus_db_fixed_data::vpc::SERVICES_VPC_ID; use nexus_types::deployment::BlueprintZoneFilter; @@ -1582,51 +1583,58 @@ impl DataStore { } // TODO: This would be nice to solve in fewer queries. - let mut subnets = HashMap::new(); - for name in subnet_names.drain() { - if let Ok((.., subnet)) = db::lookup::LookupPath::new(opctx, self) - .vpc_id(authz_vpc.id()) - .vpc_subnet_name(Name::ref_cast(&name)) - .fetch() - .await - { - subnets.insert(name, subnet); - } - } - let mut vpcs = HashMap::new(); - for name in vpc_names.drain() { - if let Ok((.., vpc)) = db::lookup::LookupPath::new(opctx, self) - .project_id(authz_project.id()) - .vpc_name(Name::ref_cast(&name)) - .fetch() - .await - { - vpcs.insert(name, vpc); - } - } - let mut instances = HashMap::new(); - for name in instance_names.drain() { - if let Ok((.., authz_instance, instance)) = + let subnets = stream::iter(subnet_names) + .filter_map(|name| async { + db::lookup::LookupPath::new(opctx, self) + .vpc_id(authz_vpc.id()) + .vpc_subnet_name(Name::ref_cast(&name)) + .fetch() + .await + .ok() + .map(|(.., subnet)| (name, subnet)) + }) + .collect::>() + .await; + + // TODO: unused until VPC peering. + let _vpcs = stream::iter(vpc_names) + .filter_map(|name| async { + db::lookup::LookupPath::new(opctx, self) + .project_id(authz_project.id()) + .vpc_name(Name::ref_cast(&name)) + .fetch() + .await + .ok() + .map(|(.., vpc)| (name, vpc)) + }) + .collect::>() + .await; + + let instances = stream::iter(instance_names) + .filter_map(|name| async { db::lookup::LookupPath::new(opctx, self) .project_id(authz_project.id()) .instance_name(Name::ref_cast(&name)) .fetch() .await - { + .ok() + .map(|(.., auth, inst)| (name, auth, inst)) + }) + .filter_map(|(name, authz_instance, instance)| async move { // XXX: currently an instance can have one primary NIC, // and it is not dual-stack (v4 + v6). We need // to clarify what should be resolved in the v6 case. - if let Ok(primary_nic) = self - .instance_get_primary_network_interface( - opctx, - &authz_instance, - ) - .await - { - instances.insert(name, (instance, primary_nic)); - } - } - } + self.instance_get_primary_network_interface( + opctx, + &authz_instance, + ) + .await + .ok() + .map(|primary_nic| (name, (instance, primary_nic))) + }) + .collect::>() + .await; + // TODO: validate names of Internet Gateways. // See the discussion in `resolve_firewall_rules_for_sled_agent` on From 28cb73985dd54bf787c8b0bdc5db0e07c7f0177a Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 11:43:03 +0100 Subject: [PATCH 49/59] One or two UUID-types missed... --- nexus/tests/integration_tests/vpc_routers.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nexus/tests/integration_tests/vpc_routers.rs b/nexus/tests/integration_tests/vpc_routers.rs index 86d43bb66a..d85a8cba8e 100644 --- a/nexus/tests/integration_tests/vpc_routers.rs +++ b/nexus/tests/integration_tests/vpc_routers.rs @@ -36,6 +36,8 @@ use omicron_common::api::external::NameOrId; use omicron_common::api::external::SimpleIdentity; use omicron_common::api::internal::shared::ResolvedVpcRoute; use omicron_common::api::internal::shared::RouterTarget; +use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::InstanceUuid; use std::collections::HashMap; pub const PROJECT_NAME: &str = "cartographer"; @@ -512,7 +514,11 @@ async fn test_vpc_routers_custom_delivered_to_instance( true, ) .await; - instance_simulate(nexus, &instance.identity.id).await; + instance_simulate( + nexus, + &InstanceUuid::from_untyped_uuid(instance.identity.id), + ) + .await; let (.., authz_instance) = LookupPath::new(&opctx, &datastore) .instance_id(instance.identity.id) From 992b9b51501b17bd061e757a6a722d8692164c90 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 12:19:57 +0100 Subject: [PATCH 50/59] More liberal use of `#[serde(default)]` --- common/src/api/external/mod.rs | 1 + common/src/api/internal/shared.rs | 3 +- illumos-utils/src/opte/port_manager.rs | 58 +++++++++---------- nexus/db-model/src/network_interface.rs | 4 +- nexus/db-model/src/omicron_zone_config.rs | 2 +- .../src/db/datastore/network_interface.rs | 4 +- nexus/db-queries/src/db/datastore/rack.rs | 18 +++--- .../execution/src/external_networking.rs | 6 +- .../planning/src/blueprint_builder/builder.rs | 2 +- nexus/test-utils/src/lib.rs | 4 +- nexus/types/src/external_api/params.rs | 1 + openapi/nexus-internal.json | 2 +- openapi/nexus.json | 10 ++-- openapi/sled-agent.json | 2 +- sled-agent/src/rack_setup/plan/service.rs | 6 +- sled-agent/src/sim/server.rs | 4 +- 16 files changed, 61 insertions(+), 66 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 16bb25f81a..67bf3380b8 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1982,6 +1982,7 @@ pub struct InstanceNetworkInterface { /// A set of additional networks that this interface may send and /// receive traffic on. + #[serde(default)] pub transit_ips: Vec, } diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index c85b04b344..b91da1ffa8 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -57,7 +57,8 @@ pub struct NetworkInterface { pub vni: Vni, pub primary: bool, pub slot: u8, - pub transit_ips: Option>, + #[serde(default)] + pub transit_ips: Vec, } /// An IP address and port range used for source NAT, i.e., making diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index f3775fc07c..984e3c55fa 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -444,37 +444,35 @@ impl PortManager { // TODO: Currently set only in initial state. // This, external IPs, and cfg'able state // (DHCP?) are probably worth being managed by an RPW. - if let Some(blocks) = &nic.transit_ips { - for block in blocks { - #[cfg(target_os = "illumos")] - { - use oxide_vpc::api::Direction; - - // In principle if this were an operation on an existing - // port, we would explicitly undo the In addition if the - // Out addition fails. - // However, failure here will just destroy the port - // outright -- this should only happen if an excessive - // number of rules are specified. - hdl.allow_cidr( - &port_name, - super::net_to_cidr(*block), - Direction::In, - )?; - hdl.allow_cidr( - &port_name, - super::net_to_cidr(*block), - Direction::Out, - )?; - } - - debug!( - self.inner.log, - "Added CIDR to in/out allowlist"; - "port_name" => &port_name, - "cidr" => ?block, - ); + for block in &nic.transit_ips { + #[cfg(target_os = "illumos")] + { + use oxide_vpc::api::Direction; + + // In principle if this were an operation on an existing + // port, we would explicitly undo the In addition if the + // Out addition fails. + // However, failure here will just destroy the port + // outright -- this should only happen if an excessive + // number of rules are specified. + hdl.allow_cidr( + &port_name, + super::net_to_cidr(*block), + Direction::In, + )?; + hdl.allow_cidr( + &port_name, + super::net_to_cidr(*block), + Direction::Out, + )?; } + + debug!( + self.inner.log, + "Added CIDR to in/out allowlist"; + "port_name" => &port_name, + "cidr" => ?block, + ); } info!( diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs index c9d05815f3..79b16b5658 100644 --- a/nexus/db-model/src/network_interface.rs +++ b/nexus/db-model/src/network_interface.rs @@ -105,9 +105,7 @@ impl NetworkInterface { vni: external::Vni::try_from(0).unwrap(), primary: self.primary, slot: *self.slot, - transit_ips: Some( - self.transit_ips.into_iter().map(Into::into).collect(), - ), + transit_ips: self.transit_ips.into_iter().map(Into::into).collect(), } } } diff --git a/nexus/db-model/src/omicron_zone_config.rs b/nexus/db-model/src/omicron_zone_config.rs index 7f269c55e3..3b18a749a7 100644 --- a/nexus/db-model/src/omicron_zone_config.rs +++ b/nexus/db-model/src/omicron_zone_config.rs @@ -659,7 +659,7 @@ impl OmicronZoneNic { vni: omicron_common::api::external::Vni::try_from(*self.vni) .context("parsing VNI")?, subnet: self.subnet.into(), - transit_ips: None, + transit_ips: vec![], }) } } diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index c7ca213327..c5a8992cd2 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -93,9 +93,7 @@ impl From for omicron_common::api::internal::shared::NetworkInterface { vni: nic.vni.0, primary: nic.primary, slot: u8::try_from(nic.slot).unwrap(), - transit_ips: Some( - nic.transit_ips.iter().map(|v| (*v).into()).collect(), - ), + transit_ips: nic.transit_ips.iter().map(|v| (*v).into()).collect(), } } } diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 0e404e1d30..627f1f60ab 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1390,7 +1390,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, }, ), @@ -1417,7 +1417,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), @@ -1464,7 +1464,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, }, ), @@ -1491,7 +1491,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, external_ip: OmicronZoneExternalSnatIp { id: ExternalIpUuid::new_v4(), @@ -1719,7 +1719,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, }, ), @@ -1751,7 +1751,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, }, ), @@ -1990,7 +1990,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, }, ), @@ -2096,7 +2096,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, }, ), @@ -2128,7 +2128,7 @@ mod test { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, }, ), diff --git a/nexus/reconfigurator/execution/src/external_networking.rs b/nexus/reconfigurator/execution/src/external_networking.rs index 6ed2cb320c..3ac1de96d5 100644 --- a/nexus/reconfigurator/execution/src/external_networking.rs +++ b/nexus/reconfigurator/execution/src/external_networking.rs @@ -499,7 +499,7 @@ mod tests { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }; let dns_id = OmicronZoneUuid::new_v4(); @@ -525,7 +525,7 @@ mod tests { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }; // Boundary NTP: @@ -554,7 +554,7 @@ mod tests { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }; Self { diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index a8979d26d0..b9de68847b 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -828,7 +828,7 @@ impl<'a> BlueprintBuilder<'a> { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], } }; diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 1570369506..7d69e6b3b0 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -688,7 +688,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { slot: 0, subnet: (*NEXUS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, - transit_ips: None, + transit_ips: vec![], }, }), }); @@ -1049,7 +1049,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { slot: 0, subnet: (*DNS_OPTE_IPV4_SUBNET).into(), vni: Vni::SERVICES_VNI, - transit_ips: None, + transit_ips: vec![], }, }, ), diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index af16780585..6e63e2b2e4 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -811,6 +811,7 @@ pub struct InstanceNetworkInterfaceUpdate { /// A set of additional networks that this interface may send and /// receive traffic on. + #[serde(default)] pub transit_ips: Vec, } diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index ee96730b7c..f6b0ab5530 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -3557,7 +3557,7 @@ "$ref": "#/components/schemas/IpNet" }, "transit_ips": { - "nullable": true, + "default": [], "type": "array", "items": { "$ref": "#/components/schemas/IpNet" diff --git a/openapi/nexus.json b/openapi/nexus.json index ce06a0f0ce..33af51bca0 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -14668,6 +14668,7 @@ }, "transit_ips": { "description": "A set of additional networks that this interface may send and receive traffic on.", + "default": [], "type": "array", "items": { "$ref": "#/components/schemas/IpNet" @@ -14690,7 +14691,6 @@ "subnet_id", "time_created", "time_modified", - "transit_ips", "vpc_id" ] }, @@ -14835,15 +14835,13 @@ }, "transit_ips": { "description": "A set of additional networks that this interface may send and receive traffic on.", + "default": [], "type": "array", "items": { "$ref": "#/components/schemas/IpNet" } } - }, - "required": [ - "transit_ips" - ] + } }, "InstanceResultsPage": { "description": "A single page of results", @@ -15817,7 +15815,7 @@ "$ref": "#/components/schemas/IpNet" }, "transit_ips": { - "nullable": true, + "default": [], "type": "array", "items": { "$ref": "#/components/schemas/IpNet" diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index fab49b332d..ab6a29653b 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -3590,7 +3590,7 @@ "$ref": "#/components/schemas/IpNet" }, "transit_ips": { - "nullable": true, + "default": [], "type": "array", "items": { "$ref": "#/components/schemas/IpNet" diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index b7e09e8f18..f13c15723c 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -1042,7 +1042,7 @@ impl ServicePortBuilder { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }; Some((nic, external_ip)) @@ -1083,7 +1083,7 @@ impl ServicePortBuilder { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }; Ok((nic, external_ip)) @@ -1141,7 +1141,7 @@ impl ServicePortBuilder { vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }; Ok((nic, snat_cfg)) diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 60c34bc33f..5b66342a1a 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -410,7 +410,7 @@ pub async fn run_standalone_server( vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, external_tls: false, external_dns_servers: vec![], @@ -454,7 +454,7 @@ pub async fn run_standalone_server( vni: Vni::SERVICES_VNI, primary: true, slot: 0, - transit_ips: None, + transit_ips: vec![], }, }, }); From ab7223b1256cc5668ab5d3481ddb11455215f8af Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 12:32:27 +0100 Subject: [PATCH 51/59] Accidentally a line. --- nexus/db-model/src/schema_versions.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 48813ea691..0ca16498ba 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -30,7 +30,6 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), KnownVersion::new(79, "nic-spoof-allow"), - KnownVersion::new(75, "vpc-subnet-routing"), KnownVersion::new(78, "vpc-subnet-routing"), KnownVersion::new(77, "remove-view-for-v2p-mappings"), KnownVersion::new(76, "lookup-region-snapshot-by-snapshot-id"), From 2568e2cdfbcf31393a6d70ee443214e16b1d3d01 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 13:10:10 +0100 Subject: [PATCH 52/59] Especially good at EXPECTORATE-ing --- .../planning/tests/output/planner_nonprovisionable_2_2a.txt | 2 +- schema/all-zone-requests.json | 6 ++---- schema/rss-service-plan-v3.json | 6 ++---- sled-agent/tests/output/new-zones-ledgers/rack2-sled11.json | 2 +- sled-agent/tests/output/new-zones-ledgers/rack2-sled12.json | 4 ++-- sled-agent/tests/output/new-zones-ledgers/rack2-sled21.json | 4 ++-- sled-agent/tests/output/new-zones-ledgers/rack2-sled25.json | 2 +- sled-agent/tests/output/new-zones-ledgers/rack2-sled8.json | 2 +- sled-agent/tests/output/new-zones-ledgers/rack3-sled11.json | 2 +- sled-agent/tests/output/new-zones-ledgers/rack3-sled13.json | 2 +- sled-agent/tests/output/new-zones-ledgers/rack3-sled14.json | 2 +- sled-agent/tests/output/new-zones-ledgers/rack3-sled15.json | 2 +- sled-agent/tests/output/new-zones-ledgers/rack3-sled20.json | 2 +- sled-agent/tests/output/new-zones-ledgers/rack3-sled25.json | 2 +- sled-agent/tests/output/new-zones-ledgers/rack3-sled8.json | 2 +- 15 files changed, 19 insertions(+), 23 deletions(-) diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt index ab6e000e54..837cc56553 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt @@ -206,7 +206,7 @@ ERRORS: ), primary: true, slot: 0, - transit_ips: None, + transit_ips: [], }, external_tls: false, external_dns_servers: [], diff --git a/schema/all-zone-requests.json b/schema/all-zone-requests.json index 7f812da1d2..329aea61ad 100644 --- a/schema/all-zone-requests.json +++ b/schema/all-zone-requests.json @@ -271,10 +271,8 @@ "$ref": "#/definitions/IpNet" }, "transit_ips": { - "type": [ - "array", - "null" - ], + "default": [], + "type": "array", "items": { "$ref": "#/definitions/IpNet" } diff --git a/schema/rss-service-plan-v3.json b/schema/rss-service-plan-v3.json index a569a4aa4f..31250148ec 100644 --- a/schema/rss-service-plan-v3.json +++ b/schema/rss-service-plan-v3.json @@ -269,10 +269,8 @@ "$ref": "#/definitions/IpNet" }, "transit_ips": { - "type": [ - "array", - "null" - ], + "default": [], + "type": "array", "items": { "$ref": "#/definitions/IpNet" } diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled11.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled11.json index b412cfe391..f3d0f3f8f1 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled11.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled11.json @@ -110,7 +110,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] } } }, diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled12.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled12.json index 550c040001..48a5b21f81 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled12.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled12.json @@ -149,7 +149,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "external_tls": true, "external_dns_servers": [ @@ -202,7 +202,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "snat_cfg": { "ip": "172.20.26.6", diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled21.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled21.json index ddccae6943..7f0a50edae 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled21.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled21.json @@ -51,7 +51,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "external_tls": true, "external_dns_servers": [ @@ -202,7 +202,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "snat_cfg": { "ip": "172.20.26.7", diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled25.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled25.json index ccfcdb7857..22a915d44b 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled25.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled25.json @@ -138,7 +138,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] } } }, diff --git a/sled-agent/tests/output/new-zones-ledgers/rack2-sled8.json b/sled-agent/tests/output/new-zones-ledgers/rack2-sled8.json index 6f6c3cd2eb..bc621aac00 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack2-sled8.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack2-sled8.json @@ -51,7 +51,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "external_tls": true, "external_dns_servers": [ diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled11.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled11.json index 79de32e47e..5cdfa60265 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled11.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled11.json @@ -170,7 +170,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "snat_cfg": { "ip": "45.154.216.39", diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled13.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled13.json index f518292390..b823814ca0 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled13.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled13.json @@ -170,7 +170,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "snat_cfg": { "ip": "45.154.216.38", diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled14.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled14.json index b2e8a7a68c..9e9441717c 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled14.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled14.json @@ -65,7 +65,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "external_tls": true, "external_dns_servers": [ diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled15.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled15.json index 776827fd38..5254aaa066 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled15.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled15.json @@ -26,7 +26,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] } } }, diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled20.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled20.json index 55b7a99d30..5635f4eb56 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled20.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled20.json @@ -135,7 +135,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "external_tls": true, "external_dns_servers": [ diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled25.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled25.json index 0efbf3693e..ae408a94ef 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled25.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled25.json @@ -82,7 +82,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] } } }, diff --git a/sled-agent/tests/output/new-zones-ledgers/rack3-sled8.json b/sled-agent/tests/output/new-zones-ledgers/rack3-sled8.json index 1e4eebaf6f..07ce46e491 100644 --- a/sled-agent/tests/output/new-zones-ledgers/rack3-sled8.json +++ b/sled-agent/tests/output/new-zones-ledgers/rack3-sled8.json @@ -65,7 +65,7 @@ "vni": 100, "primary": true, "slot": 0, - "transit_ips": null + "transit_ips": [] }, "external_tls": true, "external_dns_servers": [ From d747d01467d4be7797a562f0956489d0541c3f72 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 14:34:10 +0100 Subject: [PATCH 53/59] Again. --- schema/all-zones-requests.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/schema/all-zones-requests.json b/schema/all-zones-requests.json index 5bb511b4e0..60e0995337 100644 --- a/schema/all-zones-requests.json +++ b/schema/all-zones-requests.json @@ -155,10 +155,8 @@ "$ref": "#/definitions/IpNet" }, "transit_ips": { - "type": [ - "array", - "null" - ], + "default": [], + "type": "array", "items": { "$ref": "#/definitions/IpNet" } From c1e5e7562a8a6447baf8061c622f23449f8ce5cd Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Fri, 21 Jun 2024 19:41:21 +0100 Subject: [PATCH 54/59] Review feedback: clearer explanation of `RouteDestination` --- common/src/api/external/mod.rs | 6 +++--- nexus/types/src/external_api/params.rs | 3 ++- openapi/nexus.json | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 67bf3380b8..f0867cb5c2 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1407,14 +1407,14 @@ pub struct RouterRoute { /// common identifying metadata #[serde(flatten)] pub identity: IdentityMetadata, - /// The ID of the VPC Router to which the route belongs pub vpc_router_id: Uuid, - /// Describes the kind of router. Set at creation. `read-only` pub kind: RouterRouteKind, - + /// The location that matched packets should be forwarded to. pub target: RouteTarget, + /// The set of destination IP addresses or subnets that this route + /// will match packets against. pub destination: RouteDestination, } diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 6e63e2b2e4..19048604ad 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -1271,7 +1271,8 @@ pub struct RouterRouteCreate { pub identity: IdentityMetadataCreateParams, /// The location that matched packets should be forwarded to. pub target: RouteTarget, - /// A CIDR block (or named subnet) which this route will apply to. + /// The set of destination IP addresses or subnets that this route + /// will match packets against. pub destination: RouteDestination, } diff --git a/openapi/nexus.json b/openapi/nexus.json index 33af51bca0..868680a7c2 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -16749,7 +16749,12 @@ "type": "string" }, "destination": { - "$ref": "#/components/schemas/RouteDestination" + "description": "The set of destination IP addresses or subnets that this route will match packets against.", + "allOf": [ + { + "$ref": "#/components/schemas/RouteDestination" + } + ] }, "id": { "description": "unique, immutable, system-controlled identifier for each resource", @@ -16773,7 +16778,12 @@ ] }, "target": { - "$ref": "#/components/schemas/RouteTarget" + "description": "The location that matched packets should be forwarded to.", + "allOf": [ + { + "$ref": "#/components/schemas/RouteTarget" + } + ] }, "time_created": { "description": "timestamp when this resource was created", @@ -16811,7 +16821,7 @@ "type": "string" }, "destination": { - "description": "A CIDR block (or named subnet) which this route will apply to.", + "description": "The set of destination IP addresses or subnets that this route will match packets against.", "allOf": [ { "$ref": "#/components/schemas/RouteDestination" From 0ab64837a2dd23bf1f8c914b5c8f5d4c6e9a290e Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 25 Jun 2024 09:57:37 +0100 Subject: [PATCH 55/59] Slight adjustments to `destination` docstrings --- common/src/api/external/mod.rs | 3 +-- nexus/types/src/external_api/params.rs | 5 +++-- openapi/nexus.json | 18 ++++++++++++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index f0867cb5c2..556a7504b5 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -1413,8 +1413,7 @@ pub struct RouterRoute { pub kind: RouterRouteKind, /// The location that matched packets should be forwarded to. pub target: RouteTarget, - /// The set of destination IP addresses or subnets that this route - /// will match packets against. + /// Selects which traffic this routing rule will apply to. pub destination: RouteDestination, } diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 19048604ad..6d92f2b1ba 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -1271,8 +1271,7 @@ pub struct RouterRouteCreate { pub identity: IdentityMetadataCreateParams, /// The location that matched packets should be forwarded to. pub target: RouteTarget, - /// The set of destination IP addresses or subnets that this route - /// will match packets against. + /// Selects which traffic this routing rule will apply to. pub destination: RouteDestination, } @@ -1281,7 +1280,9 @@ pub struct RouterRouteCreate { pub struct RouterRouteUpdate { #[serde(flatten)] pub identity: IdentityMetadataUpdateParams, + /// The location that matched packets should be forwarded to. pub target: RouteTarget, + /// Selects which traffic this routing rule will apply to. pub destination: RouteDestination, } diff --git a/openapi/nexus.json b/openapi/nexus.json index 868680a7c2..4496c394bf 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -16749,7 +16749,7 @@ "type": "string" }, "destination": { - "description": "The set of destination IP addresses or subnets that this route will match packets against.", + "description": "Selects which traffic this routing rule will apply to.", "allOf": [ { "$ref": "#/components/schemas/RouteDestination" @@ -16821,7 +16821,7 @@ "type": "string" }, "destination": { - "description": "The set of destination IP addresses or subnets that this route will match packets against.", + "description": "Selects which traffic this routing rule will apply to.", "allOf": [ { "$ref": "#/components/schemas/RouteDestination" @@ -16910,7 +16910,12 @@ "type": "string" }, "destination": { - "$ref": "#/components/schemas/RouteDestination" + "description": "Selects which traffic this routing rule will apply to.", + "allOf": [ + { + "$ref": "#/components/schemas/RouteDestination" + } + ] }, "name": { "nullable": true, @@ -16921,7 +16926,12 @@ ] }, "target": { - "$ref": "#/components/schemas/RouteTarget" + "description": "The location that matched packets should be forwarded to.", + "allOf": [ + { + "$ref": "#/components/schemas/RouteTarget" + } + ] } }, "required": [ From 8162a2395ac7d897903bf9df88f842dd2c9948a7 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 26 Jun 2024 10:52:02 +0100 Subject: [PATCH 56/59] Review feedback: typed RouterKind instead of Option abuse --- common/src/api/internal/shared.rs | 12 +++++- illumos-utils/src/opte/port.rs | 8 +++- nexus/src/app/background/vpc_routes.rs | 12 ++++-- nexus/tests/integration_tests/instances.rs | 10 +++-- openapi/sled-agent.json | 47 ++++++++++++++++++---- sled-agent/src/sim/sled_agent.rs | 14 ++----- 6 files changed, 75 insertions(+), 28 deletions(-) diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 40cec46f8c..090b3c3058 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -672,7 +672,17 @@ impl RouterVersion { )] pub struct RouterId { pub vni: Vni, - pub subnet: Option, + pub kind: RouterKind, +} + +/// The scope of a set of VPC router rules. +#[derive( + Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +#[serde(tag = "type", rename_all = "snake_case", content = "subnet")] +pub enum RouterKind { + System, + Custom(IpNet), } /// Version information for routes on a given VPC subnet. diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 352411c49c..a692a02304 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -9,6 +9,7 @@ use crate::opte::Vni; use macaddr::MacAddr6; use omicron_common::api::external; use omicron_common::api::internal::shared::RouterId; +use omicron_common::api::internal::shared::RouterKind; use oxnet::IpNet; use std::net::IpAddr; use std::sync::Arc; @@ -140,10 +141,13 @@ impl Port { pub fn system_router_key(&self) -> RouterId { // Unwrap safety: both of these VNI types represent validated u24s. let vni = external::Vni::try_from(self.vni().as_u32()).unwrap(); - RouterId { vni, subnet: None } + RouterId { vni, kind: RouterKind::System } } pub fn custom_router_key(&self) -> RouterId { - RouterId { subnet: Some(*self.subnet()), ..self.system_router_key() } + RouterId { + kind: RouterKind::Custom(*self.subnet()), + ..self.system_router_key() + } } } diff --git a/nexus/src/app/background/vpc_routes.rs b/nexus/src/app/background/vpc_routes.rs index f305990a22..6500b13b5b 100644 --- a/nexus/src/app/background/vpc_routes.rs +++ b/nexus/src/app/background/vpc_routes.rs @@ -15,7 +15,7 @@ use nexus_types::{ identity::Resource, }; use omicron_common::api::internal::shared::{ - ResolvedVpcRoute, ResolvedVpcRouteSet, RouterId, RouterVersion, + ResolvedVpcRoute, ResolvedVpcRouteSet, RouterId, RouterKind, RouterVersion, }; use serde_json::json; use std::collections::hash_map::Entry; @@ -164,7 +164,7 @@ impl BackgroundTask for VpcRouteManager { }; db_routers.insert( - RouterId { vni: set.id.vni, subnet: None }, + RouterId { vni: set.id.vni, kind: RouterKind::System }, system_router, ); db_routers.extend(custom_routers.iter().map( @@ -172,7 +172,9 @@ impl BackgroundTask for VpcRouteManager { ( RouterId { vni: set.id.vni, - subnet: Some(subnet.ipv4_block.0.into()), + kind: RouterKind::Custom( + subnet.ipv4_block.0.into(), + ), }, router.clone(), ) @@ -183,7 +185,9 @@ impl BackgroundTask for VpcRouteManager { ( RouterId { vni: set.id.vni, - subnet: Some(subnet.ipv6_block.0.into()), + kind: RouterKind::Custom( + subnet.ipv6_block.0.into(), + ), }, router, ) diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index f7f970c625..75ddf847bf 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -62,6 +62,7 @@ use omicron_common::api::external::NameOrId; use omicron_common::api::external::Vni; use omicron_common::api::internal::shared::ResolvedVpcRoute; use omicron_common::api::internal::shared::RouterId; +use omicron_common::api::internal::shared::RouterKind; use omicron_nexus::app::MAX_MEMORY_BYTES_PER_INSTANCE; use omicron_nexus::app::MAX_VCPU_PER_INSTANCE; use omicron_nexus::app::MIN_MEMORY_BYTES_PER_INSTANCE; @@ -4850,11 +4851,14 @@ pub async fn assert_sled_vpc_routes( let condition = || async { let vpc_routes = sled_agent.vpc_routes.lock().await; let sys_routes_found = vpc_routes.iter().any(|(id, set)| { - *id == RouterId { vni, subnet: None } && set.routes == system_routes + *id == RouterId { vni, kind: RouterKind::System } + && set.routes == system_routes }); let custom_routes_found = vpc_routes.iter().any(|(id, set)| { - *id == RouterId { vni, subnet: Some(db_subnet.ipv4_block.0.into()) } - && set.routes == custom_routes + *id == RouterId { + vni, + kind: RouterKind::Custom(db_subnet.ipv4_block.0.into()), + } && set.routes == custom_routes }); if sys_routes_found && custom_routes_found { diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 823425afd3..4f2bcd6e98 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -4382,22 +4382,55 @@ "description": "Identifier for a VPC and/or subnet.", "type": "object", "properties": { - "subnet": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] + "kind": { + "$ref": "#/components/schemas/RouterKind" }, "vni": { "$ref": "#/components/schemas/Vni" } }, "required": [ + "kind", "vni" ] }, + "RouterKind": { + "description": "The scope of a set of VPC router rules.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "system" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "subnet": { + "$ref": "#/components/schemas/IpNet" + }, + "type": { + "type": "string", + "enum": [ + "custom" + ] + } + }, + "required": [ + "subnet", + "type" + ] + } + ] + }, "RouterTarget": { "description": "The target for a given router entry.", "oneOf": [ diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 1b7211e7d8..9cb146531b 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -38,7 +38,7 @@ use omicron_common::api::internal::nexus::{ }; use omicron_common::api::internal::shared::{ RackNetworkConfig, ResolvedVpcRoute, ResolvedVpcRouteSet, - ResolvedVpcRouteState, RouterId, RouterVersion, + ResolvedVpcRouteState, RouterId, RouterKind, RouterVersion, }; use omicron_common::disk::DiskIdentity; use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid, ZpoolUuid}; @@ -368,16 +368,8 @@ impl SledAgent { let mut routes = self.vpc_routes.lock().await; for nic in &hardware.nics { let my_routers = [ - RouterId { - // system - vni: nic.vni, - subnet: None, - }, - RouterId { - // custom - vni: nic.vni, - subnet: Some(nic.subnet), - }, + RouterId { vni: nic.vni, kind: RouterKind::System }, + RouterId { vni: nic.vni, kind: RouterKind::Custom(nic.subnet) }, ]; for router in my_routers { From e1971bc9d6a74a290d63203d22a3f6cc74a44de2 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 26 Jun 2024 11:30:06 +0100 Subject: [PATCH 57/59] Bump Maghemite. --- Cargo.lock | 21 +++++++++++---------- Cargo.toml | 6 +++--- clients/ddm-admin-client/src/lib.rs | 9 ++++----- openapi/bootstrap-agent.json | 10 ++++++++++ openapi/nexus-internal.json | 10 ++++++++++ openapi/nexus.json | 1 + openapi/sled-agent.json | 10 ++++++++++ openapi/wicketd.json | 12 ++++++++++++ package-manifest.toml | 12 ++++++------ tools/maghemite_ddm_openapi_version | 4 ++-- tools/maghemite_mg_openapi_version | 4 ++-- tools/maghemite_mgd_checksums | 4 ++-- workspace-hack/Cargo.toml | 8 ++++---- 13 files changed, 77 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ab08c0e3c..8ee031fe24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1605,8 +1605,9 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=e63f6d408908b3332d7cd89a4dd44a0f980d931d#e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source = "git+https://github.com/oxidecomputer/maghemite?rev=3c3fa8482fe09a01da62fbd35efe124ea9cac9e7#3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" dependencies = [ + "oxnet", "percent-encoding", "progenitor", "reqwest", @@ -4007,7 +4008,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.52.5", + "windows-targets 0.48.5", ] [[package]] @@ -4290,7 +4291,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=e63f6d408908b3332d7cd89a4dd44a0f980d931d#e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source = "git+https://github.com/oxidecomputer/maghemite?rev=3c3fa8482fe09a01da62fbd35efe124ea9cac9e7#3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" dependencies = [ "anyhow", "chrono", @@ -6411,7 +6412,7 @@ dependencies = [ [[package]] name = "oxnet" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/oxnet?branch=main#42b4d3c77c7f5f2636cd6c4bbf37ac3eada047e0" +source = "git+https://github.com/oxidecomputer/oxnet#2612d2203effcfdcbf83778a77f1bfd03fe6ed24" dependencies = [ "ipnetwork", "schemars", @@ -8287,9 +8288,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0218ceea14babe24a4a5836f86ade86c1effbc198164e619194cb5069187e29" +checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" dependencies = [ "bytes", "chrono", @@ -8302,9 +8303,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed5a1ccce8ff962e31a165d41f6e2a2dd1245099dc4d594f5574a86cd90f4d3" +checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" dependencies = [ "proc-macro2", "quote", @@ -8489,9 +8490,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" dependencies = [ "itoa", "ryu", diff --git a/Cargo.toml b/Cargo.toml index 4c83e9229c..bf3a8c1feb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -328,8 +328,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.12" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" } multimap = "0.10.0" nexus-auth = { path = "nexus/auth" } nexus-client = { path = "clients/nexus-client" } @@ -349,7 +349,7 @@ omicron-certificates = { path = "certificates" } omicron-passwords = { path = "passwords" } omicron-workspace-hack = "0.1.0" oxlog = { path = "dev-tools/oxlog" } -oxnet = { git = "https://github.com/oxidecomputer/oxnet", branch = "main" } +oxnet = { git = "https://github.com/oxidecomputer/oxnet" } nexus-test-interface = { path = "nexus/test-interface" } nexus-test-utils-macros = { path = "nexus/test-utils-macros" } nexus-test-utils = { path = "nexus/test-utils" } diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index b926ee2971..8cd9781e1d 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -12,7 +12,7 @@ pub use ddm_admin_client::types; pub use ddm_admin_client::Error; -use ddm_admin_client::types::{Ipv6Prefix, TunnelOrigin}; +use ddm_admin_client::types::TunnelOrigin; use ddm_admin_client::Client as InnerClient; use either::Either; use omicron_common::address::Ipv6Subnet; @@ -81,8 +81,7 @@ impl Client { pub fn advertise_prefix(&self, address: Ipv6Subnet) { let me = self.clone(); tokio::spawn(async move { - let prefix = - Ipv6Prefix { addr: address.net().prefix(), len: SLED_PREFIX }; + let prefix = address.net(); retry_notify(retry_policy_internal_service_aggressive(), || async { info!( me.log, "Sending prefix to ddmd for advertisement"; @@ -130,8 +129,8 @@ impl Client { let prefixes = self.inner.get_prefixes().await?.into_inner(); Ok(prefixes.into_iter().flat_map(|(_, prefixes)| { prefixes.into_iter().flat_map(|prefix| { - let mut segments = prefix.destination.addr.segments(); - if prefix.destination.len == BOOTSTRAP_MASK + let mut segments = prefix.destination.addr().segments(); + if prefix.destination.width() == BOOTSTRAP_MASK && segments[0] == BOOTSTRAP_PREFIX { Either::Left(interfaces.iter().map(move |interface| { diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 5d175e7b09..6050939b94 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -328,6 +328,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -340,6 +341,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -437,6 +439,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -444,11 +447,13 @@ "md5_auth_key": { "nullable": true, "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": "string" }, "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -456,6 +461,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -467,6 +473,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -474,6 +481,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -1192,6 +1200,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -1234,6 +1243,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index c3cc3c059d..72731e83e8 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -1567,6 +1567,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -1579,6 +1580,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -1676,6 +1678,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1683,11 +1686,13 @@ "md5_auth_key": { "nullable": true, "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": "string" }, "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -1695,6 +1700,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1706,6 +1712,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1713,6 +1720,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -4345,6 +4353,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -5003,6 +5012,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/openapi/nexus.json b/openapi/nexus.json index 3e0fe9d75c..a17ccf1b84 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -16002,6 +16002,7 @@ "signing_keypair": { "nullable": true, "description": "request signing key pair", + "default": null, "allOf": [ { "$ref": "#/components/schemas/DerEncodedKeyPair" diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 4f2bcd6e98..3ac130c565 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -1519,6 +1519,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -1531,6 +1532,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -1628,6 +1630,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1635,11 +1638,13 @@ "md5_auth_key": { "nullable": true, "description": "Use the given key for TCP-MD5 authentication with the peer.", + "default": null, "type": "string" }, "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -1647,6 +1652,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1658,6 +1664,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -1665,6 +1672,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -4368,6 +4376,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -4817,6 +4826,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 21e8ebeedd..555b8cf44c 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -1049,6 +1049,7 @@ "checker": { "nullable": true, "description": "Checker to apply to incoming messages.", + "default": null, "type": "string" }, "originate": { @@ -1061,6 +1062,7 @@ "shaper": { "nullable": true, "description": "Shaper to apply to outgoing messages.", + "default": null, "type": "string" } }, @@ -2854,6 +2856,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -5050,6 +5053,7 @@ "vlan_id": { "nullable": true, "description": "The VLAN id (if any) associated with this address.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 @@ -5070,6 +5074,7 @@ }, "allowed_export": { "description": "Apply export policy to this peer with an allow list.", + "default": null, "allOf": [ { "$ref": "#/components/schemas/UserSpecifiedImportExportPolicy" @@ -5078,6 +5083,7 @@ }, "allowed_import": { "description": "Apply import policy to this peer with an allow list.", + "default": null, "allOf": [ { "$ref": "#/components/schemas/UserSpecifiedImportExportPolicy" @@ -5093,6 +5099,7 @@ "auth_key_id": { "nullable": true, "description": "The key identifier for authentication to use with the peer.", + "default": null, "allOf": [ { "$ref": "#/components/schemas/BgpAuthKeyId" @@ -5152,6 +5159,7 @@ "local_pref": { "nullable": true, "description": "Apply a local preference to routes received from this peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -5159,6 +5167,7 @@ "min_ttl": { "nullable": true, "description": "Require messages from a peer have a minimum IP time to live field.", + "default": null, "type": "integer", "format": "uint8", "minimum": 0 @@ -5166,6 +5175,7 @@ "multi_exit_discriminator": { "nullable": true, "description": "Apply the provided multi-exit discriminator (MED) updates sent to the peer.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -5177,6 +5187,7 @@ "remote_asn": { "nullable": true, "description": "Require that a peer has a specified ASN.", + "default": null, "type": "integer", "format": "uint32", "minimum": 0 @@ -5184,6 +5195,7 @@ "vlan_id": { "nullable": true, "description": "Associate a VLAN ID with a BGP peer session.", + "default": null, "type": "integer", "format": "uint16", "minimum": 0 diff --git a/package-manifest.toml b/package-manifest.toml index 30fc288766..797be3e4b3 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -548,10 +548,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source.commit = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "fb1a9c24f160afb3f87b57544869f2b54876b2b4bb2f6922411e2eb04ecd4a61" +source.sha256 = "63b6c74584e32f52893730e3a567da29c7f93934c38882614aad59034bdd980d" output.type = "tarball" [package.mg-ddm] @@ -564,10 +564,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source.commit = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "07a534a6dd6975cd13c629414382f6fdb8d0a11406dd626ec93c6d50d0e7e041" +source.sha256 = "b9908b81fee00d71b750f5b9a0f866c807adb0f924ab635295d28753538836f5" output.type = "zone" output.intermediate_only = true @@ -579,10 +579,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" +source.commit = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "cf7b30a650a6501030743485f29bd556e6a379333bac2b89e746e2d389423ded" +source.sha256 = "51f446933f0d8c426b15ea0845b66664da9b9a129893d12b25d7912b52f07362" output.type = "zone" output.intermediate_only = true diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index ed24b94811..569d3d7813 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="e63f6d408908b3332d7cd89a4dd44a0f980d931d" -SHA2="004e873e4120aa26460271368485266b75b7f964e5ed4dbee8fb5db4519470d7" +COMMIT="3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" +SHA2="007bfb717ccbc077c0250dee3121aeb0c5bb0d1c16795429a514fa4f8635a5ef" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index e334871b3e..de64133971 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="e63f6d408908b3332d7cd89a4dd44a0f980d931d" -SHA2="fdb33ee7425923560534672264008ef8948d227afce948ab704de092ad72157c" +COMMIT="3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" +SHA2="e4b42ab9daad90f0c561a830b62a9d17e294b4d0da0a6d44b4030929b0c37b7e" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 2577af45d3..f9d4fd4491 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="cf7b30a650a6501030743485f29bd556e6a379333bac2b89e746e2d389423ded" -MGD_LINUX_SHA256="1d5421c3229d8a4a512ba1ca35620bb9f235c7ae80518a78589214bb16ed5148" +CIDL_SHA256="51f446933f0d8c426b15ea0845b66664da9b9a129893d12b25d7912b52f07362" +MGD_LINUX_SHA256="736067394778cc4c38fecb1ca8647db3ca7ab1b5c4446f3ce2b5350379ba95b7" diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 1b21b72495..70730e8a76 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -89,11 +89,11 @@ regex-automata = { version = "0.4.6", default-features = false, features = ["dfa regex-syntax = { version = "0.8.3" } reqwest = { version = "0.11.27", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] } ring = { version = "0.17.8", features = ["std"] } -schemars = { version = "0.8.20", features = ["bytes", "chrono", "uuid1"] } +schemars = { version = "0.8.21", features = ["bytes", "chrono", "uuid1"] } scopeguard = { version = "1.2.0" } semver = { version = "1.0.23", features = ["serde"] } serde = { version = "1.0.203", features = ["alloc", "derive", "rc"] } -serde_json = { version = "1.0.117", features = ["raw_value", "unbounded_depth"] } +serde_json = { version = "1.0.118", features = ["raw_value", "unbounded_depth"] } sha2 = { version = "0.10.8", features = ["oid"] } similar = { version = "2.5.0", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } @@ -193,11 +193,11 @@ regex-automata = { version = "0.4.6", default-features = false, features = ["dfa regex-syntax = { version = "0.8.3" } reqwest = { version = "0.11.27", features = ["blocking", "cookies", "json", "rustls-tls", "stream"] } ring = { version = "0.17.8", features = ["std"] } -schemars = { version = "0.8.20", features = ["bytes", "chrono", "uuid1"] } +schemars = { version = "0.8.21", features = ["bytes", "chrono", "uuid1"] } scopeguard = { version = "1.2.0" } semver = { version = "1.0.23", features = ["serde"] } serde = { version = "1.0.203", features = ["alloc", "derive", "rc"] } -serde_json = { version = "1.0.117", features = ["raw_value", "unbounded_depth"] } +serde_json = { version = "1.0.118", features = ["raw_value", "unbounded_depth"] } sha2 = { version = "0.10.8", features = ["oid"] } similar = { version = "2.5.0", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } From 70263a2f3c662f5c699f40e473148f0102feecd2 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 26 Jun 2024 12:40:06 +0100 Subject: [PATCH 58/59] Better conflict resolution on Nexus-managed subnet route names --- nexus/db-model/src/vpc_route.rs | 27 ++++++-- nexus/db-queries/src/db/datastore/vpc.rs | 80 ++++++++++++++---------- schema/crdb/vpc-subnet-routing/up03.sql | 8 +-- 3 files changed, 73 insertions(+), 42 deletions(-) diff --git a/nexus/db-model/src/vpc_route.rs b/nexus/db-model/src/vpc_route.rs index dda7f0b785..3015df691f 100644 --- a/nexus/db-model/src/vpc_route.rs +++ b/nexus/db-model/src/vpc_route.rs @@ -128,13 +128,32 @@ impl RouterRoute { } } + /// Create a subnet routing rule for a VPC's system router. + /// + /// This defaults to use the same name as the subnet. If this would conflict + /// with the internet gateway rules, then the UUID is used instead (alongside + /// notice that a name conflict has occurred). pub fn for_subnet( route_id: Uuid, system_router_id: Uuid, subnet: Name, - ) -> Result { - let name = format!("sn-{}", subnet).parse().map_err(|_| ())?; - Ok(Self::new( + ) -> Self { + let forbidden_names = ["default-v4", "default-v6"]; + + let name = if forbidden_names.contains(&subnet.as_str()) { + // unwrap safety: a uuid is not by itself a valid name + // so prepend it with another string. + // - length constraint is <63 chars, + // - a UUID is 36 chars including hyphens, + // - "{subnet}-" is 11 chars + // - "conflict-" is 9 chars + // = 56 chars + format!("conflict-{subnet}-{route_id}").parse().unwrap() + } else { + subnet.0.clone() + }; + + Self::new( route_id, system_router_id, external::RouterRouteKind::VpcSubnet, @@ -146,7 +165,7 @@ impl RouterRoute { target: external::RouteTarget::Subnet(subnet.0.clone()), destination: external::RouteDestination::Subnet(subnet.0), }, - )) + ) } } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index a265853cb5..988f40e770 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -296,8 +296,8 @@ impl DataStore { route_id, SERVICES_VPC.system_router_id, vpc_subnet.name().clone().into(), - ) - .expect("builtin service names are short enough for route naming"); + ); + self.router_create_route(opctx, &authz_router, route) .await .map(|_| ()) @@ -1326,15 +1326,11 @@ impl DataStore { // modify other system routes like internet gateways (which are // `RouteKind::Default`). let conn = self.pool_connection_authorized(opctx).await?; - let log = opctx.log.clone(); self.transaction_retry_wrapper("vpc_subnet_route_reconcile") - .transaction(&conn, |conn| { - let log = log.clone(); - async move { - + .transaction(&conn, |conn| async move { use db::schema::router_route::dsl; - use db::schema::vpc_subnet::dsl as subnet; use db::schema::vpc::dsl as vpc; + use db::schema::vpc_subnet::dsl as subnet; let system_router_id = vpc::vpc .filter(vpc::id.eq(vpc_id)) @@ -1352,7 +1348,10 @@ impl DataStore { .await?; let current_rules: Vec = dsl::router_route - .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .filter( + dsl::kind + .eq(RouterRouteKind(ExternalRouteKind::VpcSubnet)), + ) .filter(dsl::time_deleted.is_null()) .filter(dsl::vpc_router_id.eq(system_router_id)) .select(RouterRoute::as_select()) @@ -1360,18 +1359,28 @@ impl DataStore { .await?; // Build the add/delete sets. - let expected_names: HashSet = valid_subnets.iter() + let expected_names: HashSet = valid_subnets + .iter() .map(|v| v.identity.name.clone()) .collect(); + // This checks that we have rules which *point to* the named + // subnets, rather than working with rule names (even if these + // are set to match the subnet where possible). + // Rule names are effectively randomised when someone, e.g., + // names a subnet "default-v4"/"-v6", and this prevents us + // from repeatedly adding/deleting that route. let mut found_names = HashSet::new(); let mut invalid = Vec::new(); for rule in current_rules { let id = rule.id(); match (rule.kind.0, rule.target.0) { - (ExternalRouteKind::VpcSubnet, RouteTarget::Subnet(n)) - if expected_names.contains(Name::ref_cast(&n)) => - {let _ = found_names.insert(n.into());}, + ( + ExternalRouteKind::VpcSubnet, + RouteTarget::Subnet(n), + ) if expected_names.contains(Name::ref_cast(&n)) => { + let _ = found_names.insert(n.into()); + } _ => invalid.push(id), } } @@ -1394,34 +1403,34 @@ impl DataStore { // Duplicate rules are caught here using the UNIQUE constraint // on names in a router. Only nexus can alter the system router, // so there is no risk of collision with user-specified names. + // + // Subnets named "default-v4" or "default-v6" have their rules renamed + // to include the rule UUID. for subnet in expected_names.difference(&found_names) { let route_id = Uuid::new_v4(); - // XXX this is fallible as it is based on subnet name. - // need to control this somewhere sane. - let Ok(route) = db::model::RouterRoute::for_subnet( + let route = db::model::RouterRoute::for_subnet( route_id, system_router_id, subnet.clone(), - ) else { - error!( - log, - "Reconciling VPC routes: name {} in vpc {} is too long", - subnet, - vpc_id, - ); - continue; - }; - - match Self::router_create_route_on_connection(route, &conn).await { - Err(Error::Conflict { .. }) => return Err(DieselError::RollbackTransaction), + ); + + match Self::router_create_route_on_connection(route, &conn) + .await + { + Err(Error::Conflict { .. }) => { + return Err(DieselError::RollbackTransaction) + } Err(_) => return Err(DieselError::NotFound), - _ => {}, + _ => {} } } // Verify that route set is exactly as intended, and rollback otherwise. let current_rules: Vec = dsl::router_route - .filter(dsl::kind.eq(RouterRouteKind(ExternalRouteKind::VpcSubnet))) + .filter( + dsl::kind + .eq(RouterRouteKind(ExternalRouteKind::VpcSubnet)), + ) .filter(dsl::time_deleted.is_null()) .filter(dsl::vpc_router_id.eq(system_router_id)) .select(RouterRoute::as_select()) @@ -1429,19 +1438,22 @@ impl DataStore { .await?; if current_rules.len() != expected_names.len() { - return Err(DieselError::RollbackTransaction) + return Err(DieselError::RollbackTransaction); } for rule in current_rules { match (rule.kind.0, rule.target.0) { - (ExternalRouteKind::VpcSubnet, RouteTarget::Subnet(n)) - if expected_names.contains(Name::ref_cast(&n)) => {}, + ( + ExternalRouteKind::VpcSubnet, + RouteTarget::Subnet(n), + ) if expected_names.contains(Name::ref_cast(&n)) => {} _ => return Err(DieselError::RollbackTransaction), } } Ok(()) - }}).await + }) + .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; self.vpc_increment_rpw_version(opctx, vpc_id).await diff --git a/schema/crdb/vpc-subnet-routing/up03.sql b/schema/crdb/vpc-subnet-routing/up03.sql index d256921d34..7c4cc97a80 100644 --- a/schema/crdb/vpc-subnet-routing/up03.sql +++ b/schema/crdb/vpc-subnet-routing/up03.sql @@ -51,7 +51,7 @@ INSERT INTO omicron.public.router_route target, destination ) SELECT - gen_random_uuid(), 'sn-' || vpc_subnet.name, + gen_random_uuid(), vpc_subnet.name, 'VPC Subnet route for ''' || vpc_subnet.name || '''', now(), now(), omicron.public.vpc_router.id, 'default', @@ -76,15 +76,15 @@ WITH known_ids (new_id, new_name, new_description) AS ( 'Default internet gateway route for Oxide Services' ), ( - '001de000-c470-4000-8000-000000000004', 'sn-external-dns', + '001de000-c470-4000-8000-000000000004', 'external-dns', 'Built-in VPC Subnet for Oxide service (external-dns)' ), ( - '001de000-c470-4000-8000-000000000005', 'sn-nexus', + '001de000-c470-4000-8000-000000000005', 'nexus', 'Built-in VPC Subnet for Oxide service (nexus)' ), ( - '001de000-c470-4000-8000-000000000006', 'sn-boundary-ntp', + '001de000-c470-4000-8000-000000000006', 'boundary-ntp', 'Built-in VPC Subnet for Oxide service (boundary-ntp)' ) ) From 62de75eb7cbb19d088593a6245a70c236cf02ddf Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 26 Jun 2024 20:09:01 +0100 Subject: [PATCH 59/59] Unearthed a nice li'l bug during migration on london --- nexus/db-queries/src/db/datastore/vpc.rs | 46 +++++++++++++++++++++++- schema/crdb/vpc-subnet-routing/up03.sql | 2 +- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 988f40e770..89ee1c468e 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -2482,7 +2482,7 @@ mod tests { .await; // Add another, and get another route. - let (_, sub1) = new_subnet_ez( + let (authz_sub1, sub1) = new_subnet_ez( &opctx, &datastore, &db_vpc, @@ -2536,6 +2536,50 @@ mod tests { ) .await; + // If we use a reserved name, we should be able to update the table. + let sub1 = datastore + .vpc_update_subnet( + &opctx, + &authz_sub1, + VpcSubnetUpdate { + name: Some( + "default-v4".parse::().unwrap().into(), + ), + description: None, + time_modified: Utc::now(), + }, + ) + .await + .unwrap(); + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub1], + ) + .await; + + // Ditto for adding such a route. + let (_, sub0) = new_subnet_ez( + &opctx, + &datastore, + &db_vpc, + &authz_vpc, + "default-v6", + [172, 30, 0, 0], + 22, + ) + .await; + + verify_all_subnet_routes_in_router( + &opctx, + &datastore, + db_router.id(), + &[&sub0, &sub1], + ) + .await; + db.cleanup().await.unwrap(); logctx.cleanup_successful(); } diff --git a/schema/crdb/vpc-subnet-routing/up03.sql b/schema/crdb/vpc-subnet-routing/up03.sql index 7c4cc97a80..fb4fd2324a 100644 --- a/schema/crdb/vpc-subnet-routing/up03.sql +++ b/schema/crdb/vpc-subnet-routing/up03.sql @@ -54,7 +54,7 @@ SELECT gen_random_uuid(), vpc_subnet.name, 'VPC Subnet route for ''' || vpc_subnet.name || '''', now(), now(), - omicron.public.vpc_router.id, 'default', + omicron.public.vpc_router.id, 'vpc_subnet', 'subnet:' || vpc_subnet.name, 'subnet:' || vpc_subnet.name FROM (omicron.public.vpc_subnet JOIN omicron.public.vpc