diff --git a/.config/nextest.toml b/.config/nextest.toml index ef296d7ef8..4f927d2396 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -3,7 +3,7 @@ # # The required version should be bumped up if we need new features, performance # improvements or bugfixes that are present in newer versions of nextest. -nextest-version = { required = "0.9.59", recommended = "0.9.59" } +nextest-version = { required = "0.9.59", recommended = "0.9.64" } experimental = ["setup-scripts"] diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index 6fda8bb8d7..34f81bab68 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -7,7 +7,7 @@ set -o xtrace # NOTE: This version should be in sync with the recommended version in # .config/nextest.toml. (Maybe build an automated way to pull the recommended # version in the future.) -NEXTEST_VERSION='0.9.59' +NEXTEST_VERSION='0.9.64' cargo --version rustc --version diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index e76aaccc44..f4f1e0a999 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -2,7 +2,7 @@ #: #: name = "helios / deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.25" +#: target = "lab-2.0-opte-0.27" #: output_rules = [ #: "%/var/svc/log/oxide-sled-agent:default.log*", #: "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*", diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 0605ab6883..350ab37233 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -37,7 +37,7 @@ rustc --version # trampoline global zone images. # COMMIT=$(git rev-parse HEAD) -VERSION="1.0.4-0.ci+git${COMMIT:0:11}" +VERSION="5.0.0-0.ci+git${COMMIT:0:11}" echo "$VERSION" >/work/version.txt ptime -m ./tools/install_builder_prerequisites.sh -yp diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index 70b57a450a..0d1aec4c16 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -24,7 +24,7 @@ jobs: with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@d211c4be5a95cbcd52a0870dda7d63a107a58368 # v2 + uses: taiki-e/install-action@d140130aeedb5a946a5769684d32e3a33539f226 # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date diff --git a/Cargo.lock b/Cargo.lock index a34af40dd6..ed988f4b14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,7 +33,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cipher", "cpufeatures", ] @@ -58,7 +58,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", "version_check", ] @@ -250,7 +250,7 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-bb8-diesel" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=1446f7e0c1f05f33a0581abd51fa873c7652ab61#1446f7e0c1f05f33a0581abd51fa873c7652ab61" +source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=ed7ab5ef0513ba303d33efd41d3e9e381169d59b#ed7ab5ef0513ba303d33efd41d3e9e381169d59b" dependencies = [ "async-trait", "bb8", @@ -381,7 +381,7 @@ checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "miniz_oxide", "object 0.32.1", @@ -394,12 +394,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.5" @@ -847,12 +841,6 @@ dependencies = [ "nom", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -865,7 +853,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cipher", "cpufeatures", ] @@ -1003,6 +991,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + [[package]] name = "colorchoice" version = "1.0.0" @@ -1125,7 +1119,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1190,7 +1184,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -1204,7 +1198,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", ] @@ -1214,7 +1208,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] @@ -1226,7 +1220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", "memoffset 0.9.0", "scopeguard", @@ -1238,7 +1232,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", ] @@ -1248,7 +1242,7 @@ version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1364,7 +1358,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bd9c8e659a473bce955ae5c35b116af38af11a7acb0b480e01f3ed348aeb40" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "memchr", ] @@ -1383,7 +1377,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622178105f911d937a42cdb140730ba4a3ed2becd8ae6ce39c7d28b5d75d4588" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "curve25519-dalek-derive", "digest", @@ -1482,7 +1476,7 @@ version = "5.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "hashbrown 0.14.2", "lock_api", "once_cell", @@ -1550,6 +1544,38 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" +[[package]] +name = "defmt" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2d011b2fee29fb7d659b83c43fce9a2cb4df453e16d441a51448e448f3f98" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54f0216f6c5acb5ae1a47050a6645024e6edafc2ee32d421955eccfef12ef92e" +dependencies = [ + "defmt-parser", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.32", +] + +[[package]] +name = "defmt-parser" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269924c02afd7f94bc4cecbfa5c379f6ffcf9766b3408fe63d22c728654eccd0" +dependencies = [ + "thiserror", +] + [[package]] name = "der" version = "0.7.8" @@ -1729,7 +1755,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -1882,7 +1908,7 @@ source = "git+https://github.com/oxidecomputer/dropshot?branch=main#ff87a0175a6c dependencies = [ "async-stream", "async-trait", - "base64 0.21.5", + "base64", "bytes", "camino", "chrono", @@ -1950,6 +1976,20 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature 2.1.0", + "spki", +] + [[package]] name = "ed25519" version = "1.5.3" @@ -1991,9 +2031,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "elliptic-curve" -version = "0.13.5" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968405c8fdc9b3bf4df0a6638858cc0b52462836ab6b1c87377785dd09cf1c0b" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", "crypto-bigint", @@ -2002,11 +2042,20 @@ dependencies = [ "generic-array", "group", "hkdf", + "pem-rfc7468", + "pkcs8", "rand_core 0.6.4", + "sec1", "subtle", "zeroize", ] +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + [[package]] name = "ena" version = "0.14.2" @@ -2028,7 +2077,7 @@ version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2037,7 +2086,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "base64 0.21.5", + "base64", "chrono", "http", "hyper", @@ -2154,7 +2203,7 @@ version = "3.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "rustix 0.38.25", "windows-sys 0.48.0", ] @@ -2177,14 +2226,14 @@ checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" [[package]] name = "filetime" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", ] [[package]] @@ -2455,7 +2504,7 @@ dependencies = [ name = "gateway-client" version = "0.1.0" dependencies = [ - "base64 0.21.5", + "base64", "chrono", "gateway-messages", "omicron-workspace-hack", @@ -2558,7 +2607,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -2569,7 +2618,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -2683,7 +2732,7 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ - "base64 0.21.5", + "base64", "bytes", "headers-core", "http", @@ -3065,7 +3114,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" +source = "git+https://github.com/oxidecomputer/opte?rev=24ceba1969269e4d81bda83d8968d7d7f713c46b#24ceba1969269e4d81bda83d8968d7d7f713c46b" [[package]] name = "illumos-utils" @@ -3077,7 +3126,7 @@ dependencies = [ "byteorder", "camino", "camino-tempfile", - "cfg-if 1.0.0", + "cfg-if", "crucible-smf", "futures", "ipnetwork", @@ -3280,7 +3329,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -3468,10 +3517,10 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" +source = "git+https://github.com/oxidecomputer/opte?rev=24ceba1969269e4d81bda83d8968d7d7f713c46b#24ceba1969269e4d81bda83d8968d7d7f713c46b" dependencies = [ "quote", - "syn 1.0.109", + "syn 2.0.32", ] [[package]] @@ -3561,7 +3610,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "winapi", ] @@ -3577,7 +3626,7 @@ version = "0.1.0" source = "git+https://github.com/oxidecomputer/netadm-sys#f114bd0d543d886cd453932e9f0967de57289bc2" dependencies = [ "anyhow", - "cfg-if 1.0.0", + "cfg-if", "colored", "dlpi", "libc", @@ -3856,7 +3905,7 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "downcast", "fragile", "lazy_static", @@ -3871,7 +3920,7 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "proc-macro2", "quote", "syn 1.0.109", @@ -4010,7 +4059,7 @@ dependencies = [ "async-bb8-diesel", "async-trait", "authz-macros", - "base64 0.21.5", + "base64", "bb8", "camino", "camino-tempfile", @@ -4048,9 +4097,10 @@ dependencies = [ "oso", "oximeter", "paste", - "pem 1.1.1", + "pem", "petgraph", "pq-sys", + "rand 0.8.5", "rcgen", "ref-cast", "regex", @@ -4090,7 +4140,7 @@ name = "nexus-inventory" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.21.5", + "base64", "chrono", "expectorate", "gateway-client", @@ -4170,7 +4220,7 @@ version = "0.1.0" dependencies = [ "anyhow", "api_identity", - "base64 0.21.5", + "base64", "chrono", "dns-service-client", "futures", @@ -4203,7 +4253,7 @@ version = "0.26.2" source = "git+https://github.com/jgallagher/nix?branch=r0.26-illumos#c1a3636db0524f194b714cfd117cd9b637b8b10e" dependencies = [ "bitflags 1.3.2", - "cfg-if 1.0.0", + "cfg-if", "libc", "memoffset 0.7.1", "pin-utils", @@ -4550,7 +4600,7 @@ name = "omicron-gateway" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.21.5", + "base64", "clap 4.4.3", "dropshot", "expectorate", @@ -4594,7 +4644,7 @@ dependencies = [ "assert_matches", "async-bb8-diesel", "async-trait", - "base64 0.21.5", + "base64", "buf-list", "camino", "cancel-safe-futures", @@ -4655,7 +4705,7 @@ dependencies = [ "oximeter-producer", "parse-display", "paste", - "pem 1.1.1", + "pem", "petgraph", "pq-sys", "pretty_assertions", @@ -4666,7 +4716,7 @@ dependencies = [ "ref-cast", "regex", "reqwest", - "ring 0.16.20", + "ring 0.17.7", "rustls", "samael", "schemars", @@ -4757,7 +4807,7 @@ dependencies = [ "petgraph", "rayon", "reqwest", - "ring 0.16.20", + "ring 0.17.7", "semver 1.0.20", "serde", "sled-hardware", @@ -4804,14 +4854,14 @@ dependencies = [ "anyhow", "assert_matches", "async-trait", - "base64 0.21.5", + "base64", "bootstore", "bootstrap-agent-client", "bytes", "camino", "camino-tempfile", "cancel-safe-futures", - "cfg-if 1.0.0", + "cfg-if", "chrono", "clap 4.4.3", "crucible-agent-client", @@ -4899,11 +4949,11 @@ dependencies = [ "libc", "omicron-common", "omicron-workspace-hack", - "pem 1.1.1", + "pem", "rcgen", "regex", "reqwest", - "ring 0.16.20", + "ring 0.17.7", "rustls", "slog", "subprocess", @@ -4920,6 +4970,7 @@ name = "omicron-workspace-hack" version = "0.1.0" dependencies = [ "anyhow", + "base16ct", "bit-set", "bit-vec", "bitflags 1.3.2", @@ -4938,10 +4989,13 @@ dependencies = [ "crossbeam-utils", "crossterm", "crypto-common", + "der", "diesel", "digest", "either", + "elliptic-curve", "errno", + "ff", "flate2", "futures", "futures-channel", @@ -4953,8 +5007,10 @@ dependencies = [ "gateway-messages", "generic-array", "getrandom 0.2.10", + "group", "hashbrown 0.13.2", "hex", + "hmac", "hyper", "hyper-rustls", "indexmap 2.1.0", @@ -4974,6 +5030,7 @@ dependencies = [ "num-traits", "once_cell", "openapiv3", + "pem-rfc7468", "petgraph", "postgres-types", "ppv-lite86", @@ -4985,14 +5042,13 @@ dependencies = [ "regex-automata 0.4.3", "regex-syntax 0.8.2", "reqwest", - "ring 0.16.20", + "ring 0.17.7", "rustix 0.38.25", "schemars", "semver 1.0.20", "serde", "serde_json", "sha2", - "signature 2.1.0", "similar", "slog", "snafu", @@ -5050,9 +5106,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" @@ -5096,7 +5152,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ "bitflags 2.4.0", - "cfg-if 1.0.0", + "cfg-if", "foreign-types 0.3.2", "libc", "once_cell", @@ -5136,37 +5192,35 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" +source = "git+https://github.com/oxidecomputer/opte?rev=24ceba1969269e4d81bda83d8968d7d7f713c46b#24ceba1969269e4d81bda83d8968d7d7f713c46b" dependencies = [ - "cfg-if 0.1.10", + "cfg-if", "dyn-clone", "illumos-sys-hdrs", "kstat-macro", "opte-api", "postcard", "serde", - "smoltcp 0.8.2", + "smoltcp 0.10.0", "version_check", - "zerocopy 0.6.4", ] [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" +source = "git+https://github.com/oxidecomputer/opte?rev=24ceba1969269e4d81bda83d8968d7d7f713c46b#24ceba1969269e4d81bda83d8968d7d7f713c46b" dependencies = [ - "cfg-if 0.1.10", "illumos-sys-hdrs", "ipnetwork", "postcard", "serde", - "smoltcp 0.8.2", + "smoltcp 0.10.0", ] [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" +source = "git+https://github.com/oxidecomputer/opte?rev=24ceba1969269e4d81bda83d8968d7d7f713c46b#24ceba1969269e4d81bda83d8968d7d7f713c46b" dependencies = [ "libc", "libnet", @@ -5219,7 +5273,7 @@ name = "oxide-client" version = "0.1.0" dependencies = [ "anyhow", - "base64 0.21.5", + "base64", "chrono", "futures", "http", @@ -5240,14 +5294,13 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=258a8b59902dd36fc7ee5425e6b1fb5fc80d4649#258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" +source = "git+https://github.com/oxidecomputer/opte?rev=24ceba1969269e4d81bda83d8968d7d7f713c46b#24ceba1969269e4d81bda83d8968d7d7f713c46b" dependencies = [ - "cfg-if 0.1.10", "illumos-sys-hdrs", "opte", "serde", - "smoltcp 0.8.2", - "zerocopy 0.6.4", + "smoltcp 0.10.0", + "zerocopy 0.7.26", ] [[package]] @@ -5366,7 +5419,7 @@ dependencies = [ name = "oximeter-instruments" version = "0.1.0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "chrono", "dropshot", "futures", @@ -5414,6 +5467,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + [[package]] name = "packed_struct" version = "0.10.1" @@ -5474,7 +5539,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "instant", "libc", "redox_syscall 0.2.16", @@ -5488,7 +5553,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall 0.3.5", "smallvec 1.11.0", @@ -5594,22 +5659,13 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - [[package]] name = "pem" version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923" dependencies = [ - "base64 0.21.5", + "base64", "serde", ] @@ -5837,7 +5893,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52cff9d1d4dee5fe6d03729099f4a310a41179e0a10dbf542039873f2e826fb" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "opaque-debug", "universal-hash", @@ -5851,27 +5907,22 @@ checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" [[package]] name = "postcard" -version = "0.7.3" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a25c0b0ae06fcffe600ad392aabfa535696c8973f2253d9ac83171924c58a858" +checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8" dependencies = [ - "postcard-cobs", + "cobs", + "embedded-io", "serde", ] -[[package]] -name = "postcard-cobs" -version = "0.1.5-pre" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c68cb38ed13fd7bc9dd5db8f165b7c8d9c1a315104083a2b10f11354c2af97f" - [[package]] name = "postgres-protocol" version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64 0.21.5", + "base64", "byteorder", "bytes", "fallible-iterator", @@ -5995,6 +6046,15 @@ dependencies = [ "syn 2.0.32", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -6108,7 +6168,7 @@ version = "0.1.0" source = "git+https://github.com/oxidecomputer/propolis?rev=3e1d129151c3621d28ead5c6e5760693ba6e7fec#3e1d129151c3621d28ead5c6e5760693ba6e7fec" dependencies = [ "async-trait", - "base64 0.21.5", + "base64", "futures", "progenitor", "rand 0.8.5", @@ -6130,7 +6190,7 @@ source = "git+https://github.com/oxidecomputer/propolis?rev=3e1d129151c3621d28ea dependencies = [ "anyhow", "atty", - "base64 0.21.5", + "base64", "clap 4.4.3", "dropshot", "futures", @@ -6385,7 +6445,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6" dependencies = [ - "pem 3.0.2", + "pem", "ring 0.16.20", "time", "yasna", @@ -6562,7 +6622,7 @@ version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ - "base64 0.21.5", + "base64", "bytes", "encoding_rs", "futures-core", @@ -6611,6 +6671,16 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + [[package]] name = "ring" version = "0.16.20" @@ -6628,9 +6698,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.5" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", "getrandom 0.2.10", @@ -6646,7 +6716,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ - "base64 0.21.5", + "base64", "bitflags 2.4.0", "serde", "serde_derive", @@ -6705,7 +6775,7 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "glob", "proc-macro2", "quote", @@ -6728,9 +6798,9 @@ dependencies = [ [[package]] name = "russh" -version = "0.39.0" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7878311587d0353a854d5be954fbe68bdf6e77873933b484d1e45db12bb2f8cf" +checksum = "98bee7ebcce06bfc40a46b9d90205c6132d899bb9095c5ce9da3cdad8ec0833d" dependencies = [ "aes", "aes-gcm", @@ -6773,9 +6843,9 @@ dependencies = [ [[package]] name = "russh-keys" -version = "0.38.0" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "557ab9190022dff78116ebed5e391abbd3f424b06cd643dfe262346ab91ed8c9" +checksum = "3b5d5a656fe1c3024d829d054cd8c0c78dc831e4b2d4b08360569c3b38f3017f" dependencies = [ "aes", "async-trait", @@ -6795,11 +6865,13 @@ dependencies = [ "md5", "num-bigint", "num-integer", + "p256", "pbkdf2 0.11.0", "rand 0.7.3", "rand_core 0.6.4", "russh-cryptovec", "serde", + "sha1", "sha2", "thiserror", "tokio", @@ -6813,7 +6885,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5885493fdf0be6cdff808d1533ce878d21cfa49c7086fa00c66355cd9141bfc" dependencies = [ - "base64 0.21.5", + "base64", "blake2b_simd", "constant_time_eq 0.3.0", "crossbeam-utils", @@ -6896,7 +6968,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", - "ring 0.17.5", + "ring 0.17.7", "rustls-webpki", "sct", ] @@ -6919,7 +6991,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.5", + "base64", ] [[package]] @@ -6928,7 +7000,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.5", + "ring 0.17.7", "untrusted 0.9.0", ] @@ -6990,7 +7062,7 @@ name = "samael" version = "0.0.10" source = "git+https://github.com/njaremko/samael?branch=master#52028e45d11ceb7114bf0c730a9971207e965602" dependencies = [ - "base64 0.21.5", + "base64", "bindgen", "chrono", "data-encoding", @@ -7081,6 +7153,20 @@ dependencies = [ "untrusted 0.7.1", ] +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "secrecy" version = "0.8.0" @@ -7302,14 +7388,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "2.3.3" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" dependencies = [ - "base64 0.13.1", + "base64", "chrono", "hex", "indexmap 1.9.3", + "indexmap 2.1.0", "serde", "serde_json", "serde_with_macros", @@ -7318,9 +7405,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "2.3.3" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" +checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" dependencies = [ "darling 0.20.3", "proc-macro2", @@ -7372,7 +7459,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -7383,7 +7470,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -7544,7 +7631,7 @@ version = "0.1.0" dependencies = [ "anyhow", "camino", - "cfg-if 1.0.0", + "cfg-if", "futures", "illumos-devinfo", "illumos-utils", @@ -7572,7 +7659,7 @@ dependencies = [ "async-trait", "camino", "camino-tempfile", - "cfg-if 1.0.0", + "cfg-if", "derive_more", "glob", "illumos-utils", @@ -7729,24 +7816,27 @@ dependencies = [ [[package]] name = "smoltcp" -version = "0.8.2" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee34c1e1bfc7e9206cc0fb8030a90129b4e319ab53856249bb27642cab914fb3" +checksum = "7e9786ac45091b96f946693e05bfa4d8ca93e2d3341237d97a380107a6b38dea" dependencies = [ "bitflags 1.3.2", "byteorder", + "cfg-if", + "heapless", "managed", ] [[package]] name = "smoltcp" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9786ac45091b96f946693e05bfa4d8ca93e2d3341237d97a380107a6b38dea" +checksum = "8d2e3a36ac8fea7b94e666dfa3871063d6e0a5c9d5d4fec9a1a6b7b6760f0229" dependencies = [ "bitflags 1.3.2", "byteorder", - "cfg-if 1.0.0", + "cfg-if", + "defmt", "heapless", "managed", ] @@ -8181,7 +8271,7 @@ version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fastrand", "redox_syscall 0.4.1", "rustix 0.38.25", @@ -8322,7 +8412,7 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", ] @@ -8644,9 +8734,9 @@ checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" [[package]] name = "tough" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16dc5f42fc7ce7cb51eebc7a6ef91f4d69a6d41bb13f34a09674ec47e454d9b" +checksum = "49455926f64001de53ef047c2053e2f17440e412b8b1e958d4ad8a6008db7128" dependencies = [ "async-recursion", "async-trait", @@ -8659,10 +8749,10 @@ dependencies = [ "hex", "log", "olpc-cjson", - "pem 3.0.2", + "pem", "percent-encoding", "reqwest", - "ring 0.16.20", + "ring 0.17.7", "serde", "serde_json", "serde_plain", @@ -8671,7 +8761,7 @@ dependencies = [ "tokio", "tokio-util", "typed-path", - "untrusted 0.7.1", + "untrusted 0.9.0", "url", "walkdir", ] @@ -8688,7 +8778,7 @@ version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -8721,7 +8811,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c408c32e6a9dbb38037cece35740f2cf23c875d8ca134d33631cec83f74d3fe" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "data-encoding", "futures-channel", "futures-util", @@ -8742,7 +8832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" dependencies = [ "async-trait", - "cfg-if 1.0.0", + "cfg-if", "data-encoding", "enum-as-inner", "futures-channel", @@ -8766,7 +8856,7 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "futures-util", "ipconfig", "lazy_static", @@ -8788,7 +8878,7 @@ checksum = "99022f9befa6daec2a860be68ac28b1f0d9d7ccf441d8c5a695e35a58d88840d" dependencies = [ "async-trait", "bytes", - "cfg-if 1.0.0", + "cfg-if", "enum-as-inner", "futures-executor", "futures-util", @@ -8872,7 +8962,7 @@ dependencies = [ "omicron-test-utils", "omicron-workspace-hack", "rand 0.8.5", - "ring 0.16.20", + "ring 0.17.7", "serde", "serde_json", "serde_path_to_error", @@ -8917,9 +9007,9 @@ dependencies = [ [[package]] name = "typed-path" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb9d13b8242894ff21f9990082b90a6410a43dcc6029ac4227a1467853ba781" +checksum = "4a90726108dab678edab76459751e1cc7c597c3484a6384d6423191255fa641b" [[package]] name = "typenum" @@ -9303,7 +9393,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -9328,7 +9418,7 @@ version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -9502,7 +9592,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "base64 0.21.5", + "base64", "bootstrap-agent-client", "buf-list", "bytes", @@ -9655,6 +9745,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -9685,6 +9784,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -9697,6 +9811,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -9709,6 +9829,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -9721,6 +9847,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -9733,6 +9865,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -9745,6 +9883,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -9757,6 +9901,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -9769,6 +9919,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" version = "0.5.15" @@ -9784,7 +9940,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "windows-sys 0.48.0", ] @@ -9865,6 +10021,16 @@ dependencies = [ "zerocopy-derive 0.6.4", ] +[[package]] +name = "zerocopy" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +dependencies = [ + "byteorder", + "zerocopy-derive 0.7.26", +] + [[package]] name = "zerocopy-derive" version = "0.2.0" @@ -9887,6 +10053,17 @@ dependencies = [ "syn 2.0.32", ] +[[package]] +name = "zerocopy-derive" +version = "0.7.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.32", +] + [[package]] name = "zeroize" version = "1.7.0" diff --git a/Cargo.toml b/Cargo.toml index 931d885e7a..2bdd8522eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,7 +144,7 @@ api_identity = { path = "api_identity" } approx = "0.5.1" assert_matches = "1.5.0" assert_cmd = "2.0.12" -async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "1446f7e0c1f05f33a0581abd51fa873c7652ab61" } +async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "ed7ab5ef0513ba303d33efd41d3e9e381169d59b" } async-trait = "0.1.74" atomicwrites = "0.4.2" authz-macros = { path = "nexus/authz-macros" } @@ -191,7 +191,7 @@ dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", either = "1.9.0" expectorate = "1.1.0" fatfs = "0.3.6" -filetime = "0.2.22" +filetime = "0.2.23" flate2 = "1.0.28" flume = "0.11.0" foreign-types = "0.3.2" @@ -260,15 +260,15 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.9.1" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649", features = [ "api", "std" ] } -once_cell = "1.18.0" +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "24ceba1969269e4d81bda83d8968d7d7f713c46b", features = [ "api", "std" ] } +once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0-rc.1" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" openssl-probe = "0.1.5" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "24ceba1969269e4d81bda83d8968d7d7f713c46b" } oso = "0.27" owo-colors = "3.5.0" oximeter = { path = "oximeter/oximeter" } @@ -283,7 +283,7 @@ parse-display = "0.8.2" partial-io = { version = "0.5.4", features = ["proptest1", "tokio1"] } paste = "1.0.14" percent-encoding = "2.3.1" -pem = "1.1" +pem = "3.0" petgraph = "0.6.4" postgres-protocol = "0.6.6" predicates = "3.0.4" @@ -305,7 +305,7 @@ ref-cast = "1.0" regex = "1.10.2" regress = "0.7.1" reqwest = { version = "0.11", default-features = false } -ring = "0.16" +ring = "0.17.7" rpassword = "7.3.1" rstest = "0.18.2" rustfmt-wrapper = "0.2" @@ -321,7 +321,7 @@ serde_json = "1.0.108" serde_path_to_error = "0.1.14" serde_tokenstream = "0.2" serde_urlencoded = "0.7.1" -serde_with = "2.3.3" +serde_with = "3.4.0" serial_test = "0.10" sha2 = "0.10.8" sha3 = "0.10.8" @@ -373,7 +373,7 @@ tokio-util = { version = "0.7.10", features = ["io", "io-util"] } toml = "0.8.8" toml_edit = "0.21.0" topological-sort = "0.2.2" -tough = { version = "0.15", features = [ "http" ] } +tough = { version = "0.16.0", features = [ "http" ] } trust-dns-client = "0.22" trust-dns-proto = "0.22" trust-dns-resolver = "0.22" diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index db5272cd6e..50516a5da4 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -752,6 +752,7 @@ pub enum ResourceType { Zpool, Vmm, Ipv4NatEntry, + FloatingIp, } // IDENTITY METADATA diff --git a/common/src/ledger.rs b/common/src/ledger.rs index c120ab953c..71d03fa8ee 100644 --- a/common/src/ledger.rs +++ b/common/src/ledger.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use camino::{Utf8Path, Utf8PathBuf}; use serde::{de::DeserializeOwned, Serialize}; -use slog::{error, info, warn, Logger}; +use slog::{debug, info, warn, Logger}; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -88,7 +88,7 @@ impl Ledger { match T::read_from(log, &path).await { Ok(ledger) => ledgers.push(ledger), Err(err) => { - error!(log, "Failed to read ledger: {err}"; "path" => %path) + debug!(log, "Failed to read ledger: {err}"; "path" => %path) } } } @@ -184,7 +184,7 @@ pub trait Ledgerable: DeserializeOwned + Serialize + Send + Sync { err, }) } else { - warn!(log, "No ledger in {path}"); + info!(log, "No ledger in {path}"); Err(Error::NotFound) } } diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 85c55d4e61..08a783d8c8 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -2477,8 +2477,8 @@ async fn cmd_db_inventory_collections_show( eprintln!( "warning: {} collection error{} {} reported above", nerrors, + if nerrors == 1 { "" } else { "s" }, if nerrors == 1 { "was" } else { "were" }, - if nerrors == 1 { "" } else { "s" } ); } diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml index 65a6468a27..8a1f91eee8 100644 --- a/end-to-end-tests/Cargo.toml +++ b/end-to-end-tests/Cargo.toml @@ -16,8 +16,8 @@ omicron-test-utils.workspace = true oxide-client.workspace = true rand.workspace = true reqwest.workspace = true -russh = "0.39.0" -russh-keys = "0.38.0" +russh = "0.40.0" +russh-keys = "0.40.0" serde.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index f0a8d8d839..3558ef1c78 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -20,6 +20,7 @@ use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_common::api::internal::shared::SourceNatConfig; use oxide_vpc::api::AddRouterEntryReq; use oxide_vpc::api::DhcpCfg; +use oxide_vpc::api::ExternalIpCfg; use oxide_vpc::api::IpCfg; use oxide_vpc::api::IpCidr; use oxide_vpc::api::Ipv4Cfg; @@ -99,7 +100,8 @@ impl PortManager { &self, nic: &NetworkInterface, source_nat: Option, - external_ips: &[IpAddr], + ephemeral_ip: Option, + floating_ips: &[IpAddr], firewall_rules: &[VpcFirewallRule], dhcp_config: DhcpCfg, ) -> Result<(Port, PortTicket), Error> { @@ -111,13 +113,6 @@ impl PortManager { let boundary_services = default_boundary_services(); // Describe the external IP addresses for this port. - // - // Note that we're currently only taking the first address, which is all - // that OPTE supports. The array is guaranteed to be limited by Nexus. - // See https://github.com/oxidecomputer/omicron/issues/1467 - // See https://github.com/oxidecomputer/opte/issues/196 - let external_ip = external_ips.get(0); - macro_rules! ip_cfg { ($ip:expr, $log_prefix:literal, $ip_t:path, $cidr_t:path, $ipcfg_e:path, $ipcfg_t:ident, $snat_t:ident) => {{ @@ -152,25 +147,43 @@ impl PortManager { } None => None, }; - let external_ip = match external_ip { - Some($ip_t(ip)) => Some((*ip).into()), + let ephemeral_ip = match ephemeral_ip { + Some($ip_t(ip)) => Some(ip.into()), Some(_) => { error!( self.inner.log, - concat!($log_prefix, " external IP"); - "external_ip" => ?external_ip, + concat!($log_prefix, " ephemeral IP"); + "ephemeral_ip" => ?ephemeral_ip, ); return Err(Error::InvalidPortIpConfig); } None => None, }; + let floating_ips: Vec<_> = floating_ips + .iter() + .copied() + .map(|ip| match ip { + $ip_t(ip) => Ok(ip.into()), + _ => { + error!( + self.inner.log, + concat!($log_prefix, " ephemeral IP"); + "ephemeral_ip" => ?ephemeral_ip, + ); + Err(Error::InvalidPortIpConfig) + } + }) + .collect::, _>>()?; $ipcfg_e($ipcfg_t { vpc_subnet, private_ip: $ip.into(), gateway_ip: gateway_ip.into(), - snat, - external_ips: external_ip, + external_ips: ExternalIpCfg { + ephemeral_ip, + snat, + floating_ips, + }, }) }} } diff --git a/nexus/db-model/src/external_ip.rs b/nexus/db-model/src/external_ip.rs index 1152e0109c..1a755f0396 100644 --- a/nexus/db-model/src/external_ip.rs +++ b/nexus/db-model/src/external_ip.rs @@ -7,10 +7,12 @@ use crate::impl_enum_type; use crate::schema::external_ip; +use crate::schema::floating_ip; use crate::Name; use crate::SqlU16; use chrono::DateTime; use chrono::Utc; +use db_macros::Resource; use diesel::Queryable; use diesel::Selectable; use ipnetwork::IpNetwork; @@ -18,6 +20,9 @@ use nexus_types::external_api::shared; use nexus_types::external_api::views; use omicron_common::address::NUM_SOURCE_NAT_PORTS; use omicron_common::api::external::Error; +use omicron_common::api::external::IdentityMetadata; +use serde::Deserialize; +use serde::Serialize; use std::convert::TryFrom; use std::net::IpAddr; use uuid::Uuid; @@ -69,6 +74,30 @@ pub struct ExternalIp { pub ip: IpNetwork, pub first_port: SqlU16, pub last_port: SqlU16, + // Only Some(_) for instance Floating IPs + pub project_id: Option, +} + +/// A view type constructed from `ExternalIp` used to represent Floating IP +/// objects in user-facing APIs. +/// +/// This View type fills a similar niche to `ProjectImage` etc.: we need to +/// represent identity as non-nullable (ditto for parent project) so as to +/// play nicely with authz and resource APIs. +#[derive( + Queryable, Selectable, Clone, Debug, Resource, Serialize, Deserialize, +)] +#[diesel(table_name = floating_ip)] +pub struct FloatingIp { + #[diesel(embed)] + pub identity: FloatingIpIdentity, + + pub ip_pool_id: Uuid, + pub ip_pool_range_id: Uuid, + pub is_service: bool, + pub parent_id: Option, + pub ip: IpNetwork, + pub project_id: Uuid, } impl From for sled_agent_client::types::SourceNatConfig { @@ -93,6 +122,7 @@ pub struct IncompleteExternalIp { is_service: bool, parent_id: Option, pool_id: Uuid, + project_id: Option, // Optional address requesting that a specific IP address be allocated. explicit_ip: Option, // Optional range when requesting a specific SNAT range be allocated. @@ -114,6 +144,7 @@ impl IncompleteExternalIp { is_service: false, parent_id: Some(instance_id), pool_id, + project_id: None, explicit_ip: None, explicit_port_range: None, } @@ -129,6 +160,7 @@ impl IncompleteExternalIp { is_service: false, parent_id: Some(instance_id), pool_id, + project_id: None, explicit_ip: None, explicit_port_range: None, } @@ -138,6 +170,7 @@ impl IncompleteExternalIp { id: Uuid, name: &Name, description: &str, + project_id: Uuid, pool_id: Uuid, ) -> Self { Self { @@ -149,11 +182,35 @@ impl IncompleteExternalIp { is_service: false, parent_id: None, pool_id, + project_id: Some(project_id), explicit_ip: None, explicit_port_range: None, } } + pub fn for_floating_explicit( + id: Uuid, + name: &Name, + description: &str, + project_id: Uuid, + explicit_ip: IpAddr, + pool_id: Uuid, + ) -> Self { + Self { + id, + name: Some(name.clone()), + description: Some(description.to_string()), + time_created: Utc::now(), + kind: IpKind::Floating, + is_service: false, + parent_id: None, + pool_id, + project_id: Some(project_id), + explicit_ip: Some(explicit_ip.into()), + explicit_port_range: None, + } + } + pub fn for_service_explicit( id: Uuid, name: &Name, @@ -171,6 +228,7 @@ impl IncompleteExternalIp { is_service: true, parent_id: Some(service_id), pool_id, + project_id: None, explicit_ip: Some(IpNetwork::from(address)), explicit_port_range: None, } @@ -199,6 +257,7 @@ impl IncompleteExternalIp { is_service: true, parent_id: Some(service_id), pool_id, + project_id: None, explicit_ip: Some(IpNetwork::from(address)), explicit_port_range, } @@ -220,6 +279,7 @@ impl IncompleteExternalIp { is_service: true, parent_id: Some(service_id), pool_id, + project_id: None, explicit_ip: None, explicit_port_range: None, } @@ -235,6 +295,7 @@ impl IncompleteExternalIp { is_service: true, parent_id: Some(service_id), pool_id, + project_id: None, explicit_ip: None, explicit_port_range: None, } @@ -272,6 +333,10 @@ impl IncompleteExternalIp { &self.pool_id } + pub fn project_id(&self) -> &Option { + &self.project_id + } + pub fn explicit_ip(&self) -> &Option { &self.explicit_ip } @@ -308,3 +373,78 @@ impl TryFrom for views::ExternalIp { Ok(views::ExternalIp { kind, ip: ip.ip.ip() }) } } + +impl TryFrom for FloatingIp { + type Error = Error; + + fn try_from(ip: ExternalIp) -> Result { + if ip.kind != IpKind::Floating { + return Err(Error::internal_error( + "attempted to convert non-floating external IP to floating", + )); + } + if ip.is_service { + return Err(Error::internal_error( + "Service IPs should not be exposed in the API", + )); + } + + let project_id = ip.project_id.ok_or(Error::internal_error( + "database schema guarantees parent project for non-service FIP", + ))?; + + let name = ip.name.ok_or(Error::internal_error( + "database schema guarantees ID metadata for non-service FIP", + ))?; + + let description = ip.description.ok_or(Error::internal_error( + "database schema guarantees ID metadata for non-service FIP", + ))?; + + let identity = FloatingIpIdentity { + id: ip.id, + name, + description, + time_created: ip.time_created, + time_modified: ip.time_modified, + time_deleted: ip.time_deleted, + }; + + Ok(FloatingIp { + ip: ip.ip, + identity, + project_id, + ip_pool_id: ip.ip_pool_id, + ip_pool_range_id: ip.ip_pool_range_id, + is_service: ip.is_service, + parent_id: ip.parent_id, + }) + } +} + +impl TryFrom for views::FloatingIp { + type Error = Error; + + fn try_from(ip: ExternalIp) -> Result { + FloatingIp::try_from(ip).map(Into::into) + } +} + +impl From for views::FloatingIp { + fn from(ip: FloatingIp) -> Self { + let identity = IdentityMetadata { + id: ip.identity.id, + name: ip.identity.name.into(), + description: ip.identity.description, + time_created: ip.identity.time_created, + time_modified: ip.identity.time_modified, + }; + + views::FloatingIp { + ip: ip.ip.ip(), + identity, + project_id: ip.project_id, + instance_id: ip.parent_id, + } + } +} diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 373785799e..51501b4894 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -525,6 +525,7 @@ table! { time_created -> Timestamptz, time_modified -> Timestamptz, time_deleted -> Nullable, + ip_pool_id -> Uuid, ip_pool_range_id -> Uuid, is_service -> Bool, @@ -533,6 +534,26 @@ table! { ip -> Inet, first_port -> Int4, last_port -> Int4, + + project_id -> Nullable, + } +} + +table! { + floating_ip (id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + + ip_pool_id -> Uuid, + ip_pool_range_id -> Uuid, + is_service -> Bool, + parent_id -> Nullable, + ip -> Inet, + project_id -> Uuid, } } @@ -1301,7 +1322,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(18, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(19, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/nexus/db-model/src/sled.rs b/nexus/db-model/src/sled.rs index 0f6d1b911e..85a6b3139c 100644 --- a/nexus/db-model/src/sled.rs +++ b/nexus/db-model/src/sled.rs @@ -232,7 +232,7 @@ impl SledUpdate { } /// A set of constraints that can be placed on operations that select a sled. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct SledReservationConstraints { must_select_from: Vec, } diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index 94e3a56abf..9d8afd1fea 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -32,6 +32,7 @@ oso.workspace = true paste.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. pq-sys = "*" +rand.workspace = true ref-cast.workspace = true samael.workspace = true serde.workspace = true diff --git a/nexus/db-queries/src/authz/api_resources.rs b/nexus/db-queries/src/authz/api_resources.rs index b22fe1ac25..2dfe2f7174 100644 --- a/nexus/db-queries/src/authz/api_resources.rs +++ b/nexus/db-queries/src/authz/api_resources.rs @@ -791,6 +791,14 @@ authz_resource! { polar_snippet = InProject, } +authz_resource! { + name = "FloatingIp", + parent = "Project", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + // Customer network integration resources nested below "Fleet" authz_resource! { diff --git a/nexus/db-queries/src/authz/oso_generic.rs b/nexus/db-queries/src/authz/oso_generic.rs index e642062ead..6098379287 100644 --- a/nexus/db-queries/src/authz/oso_generic.rs +++ b/nexus/db-queries/src/authz/oso_generic.rs @@ -131,6 +131,7 @@ pub fn make_omicron_oso(log: &slog::Logger) -> Result { VpcRouter::init(), RouterRoute::init(), VpcSubnet::init(), + FloatingIp::init(), // Silo-level resources Image::init(), SiloImage::init(), diff --git a/nexus/db-queries/src/authz/policy_test/resources.rs b/nexus/db-queries/src/authz/policy_test/resources.rs index 3049f3b9bf..8bdd97923b 100644 --- a/nexus/db-queries/src/authz/policy_test/resources.rs +++ b/nexus/db-queries/src/authz/policy_test/resources.rs @@ -319,6 +319,13 @@ async fn make_project( Uuid::new_v4(), LookupType::ByName(image_name), )); + + let floating_ip_name = format!("{project_name}-fip1"); + builder.new_resource(authz::FloatingIp::new( + project.clone(), + Uuid::new_v4(), + LookupType::ByName(floating_ip_name), + )); } /// Returns the set of authz classes exempted from the coverage test diff --git a/nexus/db-queries/src/db/collection_attach.rs b/nexus/db-queries/src/db/collection_attach.rs index ea4d9d5beb..fccc1aa324 100644 --- a/nexus/db-queries/src/db/collection_attach.rs +++ b/nexus/db-queries/src/db/collection_attach.rs @@ -563,12 +563,9 @@ where #[cfg(test)] mod test { use super::*; - use crate::db::{ - self, error::TransactionError, identity::Resource as IdentityResource, - }; + use crate::db::{self, identity::Resource as IdentityResource}; use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - ConnectionManager, + AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, }; use chrono::Utc; use db_macros::Resource; @@ -999,22 +996,12 @@ mod test { .set(resource::dsl::collection_id.eq(collection_id)), ); - type TxnError = - TransactionError>; - let result = conn - .transaction_async(|conn| async move { - attach_query.attach_and_get_result_async(&conn).await.map_err( - |e| match e { - AttachError::DatabaseError(e) => TxnError::from(e), - e => TxnError::CustomError(e), - }, - ) - }) - .await; - // "attach_and_get_result" should return the "attached" resource. - let (returned_collection, returned_resource) = - result.expect("Attach should have worked"); + let (returned_collection, returned_resource) = attach_query + .attach_and_get_result_async(&conn) + .await + .expect("Attach should have worked"); + assert_eq!( returned_resource.collection_id.expect("Expected a collection ID"), collection_id diff --git a/nexus/db-queries/src/db/collection_detach_many.rs b/nexus/db-queries/src/db/collection_detach_many.rs index 8df6d4aed4..986cfb70b7 100644 --- a/nexus/db-queries/src/db/collection_detach_many.rs +++ b/nexus/db-queries/src/db/collection_detach_many.rs @@ -479,12 +479,9 @@ where mod test { use super::*; use crate::db::collection_attach::DatastoreAttachTarget; - use crate::db::{ - self, error::TransactionError, identity::Resource as IdentityResource, - }; + use crate::db::{self, identity::Resource as IdentityResource}; use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - ConnectionManager, + AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, }; use chrono::Utc; use db_macros::Resource; @@ -919,21 +916,12 @@ mod test { .set(resource::dsl::collection_id.eq(Option::::None)), ); - type TxnError = - TransactionError>; - let result = conn - .transaction_async(|conn| async move { - detach_query.detach_and_get_result_async(&conn).await.map_err( - |e| match e { - DetachManyError::DatabaseError(e) => TxnError::from(e), - e => TxnError::CustomError(e), - }, - ) - }) - .await; - // "detach_and_get_result" should return the "detached" resource. - let returned_collection = result.expect("Detach should have worked"); + let returned_collection = detach_query + .detach_and_get_result_async(&conn) + .await + .expect("Detach should have worked"); + // The returned values should be the latest value in the DB. assert_eq!( returned_collection, diff --git a/nexus/db-queries/src/db/datastore/address_lot.rs b/nexus/db-queries/src/db/datastore/address_lot.rs index 97dfb59eba..5c2ffbf1d0 100644 --- a/nexus/db-queries/src/db/datastore/address_lot.rs +++ b/nexus/db-queries/src/db/datastore/address_lot.rs @@ -13,9 +13,9 @@ use crate::db::error::TransactionError; use crate::db::model::Name; use crate::db::model::{AddressLot, AddressLotBlock, AddressLotReservedBlock}; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, Connection}; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::{AsyncRunQueryDsl, Connection}; use chrono::Utc; -use diesel::result::Error as DieselError; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use diesel_dtrace::DTraceConnection; use ipnetwork::IpNetwork; @@ -45,11 +45,12 @@ impl DataStore { use db::schema::address_lot::dsl as lot_dsl; use db::schema::address_lot_block::dsl as block_dsl; - self.pool_connection_authorized(opctx) - .await? - // TODO https://github.com/oxidecomputer/omicron/issues/2811 - // Audit external networking database transaction usage - .transaction_async(|conn| async move { + let conn = self.pool_connection_authorized(opctx).await?; + + // TODO https://github.com/oxidecomputer/omicron/issues/2811 + // Audit external networking database transaction usage + self.transaction_retry_wrapper("address_lot_create") + .transaction(&conn, |conn| async move { let lot = AddressLot::new(¶ms.identity, params.kind.into()); let db_lot: AddressLot = @@ -81,15 +82,14 @@ impl DataStore { Ok(AddressLotCreateResult { lot: db_lot, blocks: db_blocks }) }) .await - .map_err(|e| match e { - DieselError::DatabaseError(_, _) => public_error_from_diesel( + .map_err(|e| { + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::AddressLot, ¶ms.identity.name.as_str(), ), - ), - _ => public_error_from_diesel(e, ErrorHandler::Server), + ) }) } @@ -113,47 +113,54 @@ impl DataStore { LotInUse, } - type TxnError = TransactionError; + let err = OptionalError::new(); // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - let rsvd: Vec = - rsvd_block_dsl::address_lot_rsvd_block - .filter(rsvd_block_dsl::address_lot_id.eq(id)) - .select(AddressLotReservedBlock::as_select()) - .limit(1) - .load_async(&conn) - .await?; - - if !rsvd.is_empty() { - Err(TxnError::CustomError(AddressLotDeleteError::LotInUse))?; - } + self.transaction_retry_wrapper("address_lot_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let rsvd: Vec = + rsvd_block_dsl::address_lot_rsvd_block + .filter(rsvd_block_dsl::address_lot_id.eq(id)) + .select(AddressLotReservedBlock::as_select()) + .limit(1) + .load_async(&conn) + .await?; + + if !rsvd.is_empty() { + return Err(err.bail(AddressLotDeleteError::LotInUse)); + } + + let now = Utc::now(); + diesel::update(lot_dsl::address_lot) + .filter(lot_dsl::time_deleted.is_null()) + .filter(lot_dsl::id.eq(id)) + .set(lot_dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await?; - let now = Utc::now(); - diesel::update(lot_dsl::address_lot) - .filter(lot_dsl::time_deleted.is_null()) - .filter(lot_dsl::id.eq(id)) - .set(lot_dsl::time_deleted.eq(now)) - .execute_async(&conn) - .await?; - - diesel::delete(block_dsl::address_lot_block) - .filter(block_dsl::address_lot_id.eq(id)) - .execute_async(&conn) - .await?; - - Ok(()) - }) - .await - .map_err(|e| match e { - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - TxnError::CustomError(AddressLotDeleteError::LotInUse) => { - Error::invalid_request("lot is in use") - } - }) + diesel::delete(block_dsl::address_lot_block) + .filter(block_dsl::address_lot_id.eq(id)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + AddressLotDeleteError::LotInUse => { + Error::invalid_request("lot is in use") + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } pub async fn address_lot_list( diff --git a/nexus/db-queries/src/db/datastore/bgp.rs b/nexus/db-queries/src/db/datastore/bgp.rs index ff314a2564..28075b0ded 100644 --- a/nexus/db-queries/src/db/datastore/bgp.rs +++ b/nexus/db-queries/src/db/datastore/bgp.rs @@ -3,11 +3,11 @@ use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::Name; use crate::db::model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig}; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use nexus_types::external_api::params; @@ -30,33 +30,33 @@ impl DataStore { use db::schema::{ bgp_announce_set, bgp_announce_set::dsl as announce_set_dsl, }; - let pool = self.pool_connection_authorized(opctx).await?; - - pool.transaction_async(|conn| async move { - let id: Uuid = match &config.bgp_announce_set_id { - NameOrId::Name(name) => { - announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::time_deleted.is_null()) - .filter(bgp_announce_set::name.eq(name.to_string())) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await? - } - NameOrId::Id(id) => *id, - }; - - let config = BgpConfig::from_config_create(config, id); - - let result = diesel::insert_into(dsl::bgp_config) - .values(config.clone()) - .returning(BgpConfig::as_returning()) - .get_result_async(&conn) - .await?; - Ok(result) - }) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("bgp_config_set") + .transaction(&conn, |conn| async move { + let id: Uuid = match &config.bgp_announce_set_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::time_deleted.is_null()) + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => *id, + }; + + let config = BgpConfig::from_config_create(config, id); + + let result = diesel::insert_into(dsl::bgp_config) + .values(config.clone()) + .returning(BgpConfig::as_returning()) + .get_result_async(&conn) + .await?; + Ok(result) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn bgp_config_delete( @@ -74,54 +74,59 @@ impl DataStore { enum BgpConfigDeleteError { ConfigInUse, } - type TxnError = TransactionError; - - let pool = self.pool_connection_authorized(opctx).await?; - pool.transaction_async(|conn| async move { - let name_or_id = sel.name_or_id.clone(); - - let id: Uuid = match name_or_id { - NameOrId::Id(id) => id, - NameOrId::Name(name) => { - bgp_config_dsl::bgp_config - .filter(bgp_config::name.eq(name.to_string())) - .select(bgp_config::id) - .limit(1) - .first_async::(&conn) - .await? - } - }; - - let count = - sps_bgp_peer_config_dsl::switch_port_settings_bgp_peer_config - .filter(sps_bgp_peer_config::bgp_config_id.eq(id)) - .count() - .execute_async(&conn) - .await?; - - if count > 0 { - return Err(TxnError::CustomError( - BgpConfigDeleteError::ConfigInUse, - )); - } - diesel::update(bgp_config_dsl::bgp_config) - .filter(bgp_config_dsl::id.eq(id)) - .set(bgp_config_dsl::time_deleted.eq(Utc::now())) - .execute_async(&conn) - .await?; + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("bgp_config_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let name_or_id = sel.name_or_id.clone(); + + let id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => { + bgp_config_dsl::bgp_config + .filter(bgp_config::name.eq(name.to_string())) + .select(bgp_config::id) + .limit(1) + .first_async::(&conn) + .await? + } + }; + + let count = + sps_bgp_peer_config_dsl::switch_port_settings_bgp_peer_config + .filter(sps_bgp_peer_config::bgp_config_id.eq(id)) + .count() + .execute_async(&conn) + .await?; + + if count > 0 { + return Err(err.bail(BgpConfigDeleteError::ConfigInUse)); + } + + diesel::update(bgp_config_dsl::bgp_config) + .filter(bgp_config_dsl::id.eq(id)) + .set(bgp_config_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; - Ok(()) - }) - .await - .map_err(|e| match e { - TxnError::CustomError(BgpConfigDeleteError::ConfigInUse) => { - Error::invalid_request("BGP config in use") - } - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + BgpConfigDeleteError::ConfigInUse => { + Error::invalid_request("BGP config in use") + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } pub async fn bgp_config_get( @@ -131,7 +136,7 @@ impl DataStore { ) -> LookupResult { use db::schema::bgp_config; use db::schema::bgp_config::dsl; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let name_or_id = name_or_id.clone(); @@ -140,14 +145,14 @@ impl DataStore { .filter(bgp_config::name.eq(name.to_string())) .select(BgpConfig::as_select()) .limit(1) - .first_async::(&*pool) + .first_async::(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), NameOrId::Id(id) => dsl::bgp_config .filter(bgp_config::id.eq(id)) .select(BgpConfig::as_select()) .limit(1) - .first_async::(&*pool) + .first_async::(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), }?; @@ -162,7 +167,7 @@ impl DataStore { ) -> ListResultVec { use db::schema::bgp_config::dsl; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; match pagparams { PaginatedBy::Id(pagparams) => { @@ -176,7 +181,7 @@ impl DataStore { } .filter(dsl::time_deleted.is_null()) .select(BgpConfig::as_select()) - .load_async(&*pool) + .load_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } @@ -195,47 +200,64 @@ impl DataStore { enum BgpAnnounceListError { AnnounceSetNotFound(Name), } - type TxnError = TransactionError; - - let pool = self.pool_connection_authorized(opctx).await?; - pool.transaction_async(|conn| async move { - let name_or_id = sel.name_or_id.clone(); - - let announce_id: Uuid = match name_or_id { - NameOrId::Id(id) => id, - NameOrId::Name(name) => announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::time_deleted.is_null()) - .filter(bgp_announce_set::name.eq(name.to_string())) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError( - BgpAnnounceListError::AnnounceSetNotFound( - Name::from(name.clone()), - ), - ) - })?, - }; - - let result = announce_dsl::bgp_announcement - .filter(announce_dsl::announce_set_id.eq(announce_id)) - .select(BgpAnnouncement::as_select()) - .load_async(&conn) - .await?; - - Ok(result) - }) - .await - .map_err(|e| match e { - TxnError::CustomError( - BgpAnnounceListError::AnnounceSetNotFound(name), - ) => Error::not_found_by_name(ResourceType::BgpAnnounceSet, &name), - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("bgp_announce_list") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let name_or_id = sel.name_or_id.clone(); + + let announce_id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter( + bgp_announce_set::time_deleted.is_null(), + ) + .filter( + bgp_announce_set::name.eq(name.to_string()), + ) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or( + e, + BgpAnnounceListError::AnnounceSetNotFound( + Name::from(name.clone()), + ) + ) + })? + } + }; + + let result = announce_dsl::bgp_announcement + .filter(announce_dsl::announce_set_id.eq(announce_id)) + .select(BgpAnnouncement::as_select()) + .load_async(&conn) + .await?; + + Ok(result) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + BgpAnnounceListError::AnnounceSetNotFound(name) => { + Error::not_found_by_name( + ResourceType::BgpAnnounceSet, + &name, + ) + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } pub async fn bgp_create_announce_set( @@ -246,37 +268,39 @@ impl DataStore { use db::schema::bgp_announce_set::dsl as announce_set_dsl; use db::schema::bgp_announcement::dsl as bgp_announcement_dsl; - let pool = self.pool_connection_authorized(opctx).await?; - pool.transaction_async(|conn| async move { - let bas: BgpAnnounceSet = announce.clone().into(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("bgp_create_announce_set") + .transaction(&conn, |conn| async move { + let bas: BgpAnnounceSet = announce.clone().into(); - let db_as: BgpAnnounceSet = - diesel::insert_into(announce_set_dsl::bgp_announce_set) - .values(bas.clone()) - .returning(BgpAnnounceSet::as_returning()) - .get_result_async::(&conn) - .await?; - - let mut db_annoucements = Vec::new(); - for a in &announce.announcement { - let an = BgpAnnouncement { - announce_set_id: db_as.id(), - address_lot_block_id: bas.identity.id, - network: a.network.into(), - }; - let an = - diesel::insert_into(bgp_announcement_dsl::bgp_announcement) - .values(an.clone()) - .returning(BgpAnnouncement::as_returning()) - .get_result_async::(&conn) + let db_as: BgpAnnounceSet = + diesel::insert_into(announce_set_dsl::bgp_announce_set) + .values(bas.clone()) + .returning(BgpAnnounceSet::as_returning()) + .get_result_async::(&conn) .await?; - db_annoucements.push(an); - } - Ok((db_as, db_annoucements)) - }) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + let mut db_annoucements = Vec::new(); + for a in &announce.announcement { + let an = BgpAnnouncement { + announce_set_id: db_as.id(), + address_lot_block_id: bas.identity.id, + network: a.network.into(), + }; + let an = diesel::insert_into( + bgp_announcement_dsl::bgp_announcement, + ) + .values(an.clone()) + .returning(BgpAnnouncement::as_returning()) + .get_result_async::(&conn) + .await?; + db_annoucements.push(an); + } + + Ok((db_as, db_annoucements)) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn bgp_delete_announce_set( @@ -295,57 +319,67 @@ impl DataStore { enum BgpAnnounceSetDeleteError { AnnounceSetInUse, } - type TxnError = TransactionError; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let name_or_id = sel.name_or_id.clone(); - pool.transaction_async(|conn| async move { - let id: Uuid = match name_or_id { - NameOrId::Name(name) => { - announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::name.eq(name.to_string())) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await? - } - NameOrId::Id(id) => id, - }; - - let count = bgp_config_dsl::bgp_config - .filter(bgp_config::bgp_announce_set_id.eq(id)) - .count() - .execute_async(&conn) - .await?; - - if count > 0 { - return Err(TxnError::CustomError( - BgpAnnounceSetDeleteError::AnnounceSetInUse, - )); - } + let err = OptionalError::new(); + self.transaction_retry_wrapper("bgp_delete_announce_set") + .transaction(&conn, |conn| { + let err = err.clone(); + let name_or_id = name_or_id.clone(); + async move { + let id: Uuid = match name_or_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter( + bgp_announce_set::name.eq(name.to_string()), + ) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => id, + }; + + let count = bgp_config_dsl::bgp_config + .filter(bgp_config::bgp_announce_set_id.eq(id)) + .count() + .execute_async(&conn) + .await?; - diesel::update(announce_set_dsl::bgp_announce_set) - .filter(announce_set_dsl::id.eq(id)) - .set(announce_set_dsl::time_deleted.eq(Utc::now())) - .execute_async(&conn) - .await?; + if count > 0 { + return Err(err.bail( + BgpAnnounceSetDeleteError::AnnounceSetInUse, + )); + } - diesel::delete(bgp_announcement_dsl::bgp_announcement) - .filter(bgp_announcement_dsl::announce_set_id.eq(id)) - .execute_async(&conn) - .await?; + diesel::update(announce_set_dsl::bgp_announce_set) + .filter(announce_set_dsl::id.eq(id)) + .set(announce_set_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; - Ok(()) - }) - .await - .map_err(|e| match e { - TxnError::CustomError( - BgpAnnounceSetDeleteError::AnnounceSetInUse, - ) => Error::invalid_request("BGP announce set in use"), - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + diesel::delete(bgp_announcement_dsl::bgp_announcement) + .filter(bgp_announcement_dsl::announce_set_id.eq(id)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + BgpAnnounceSetDeleteError::AnnounceSetInUse => { + Error::invalid_request("BGP announce set in use") + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } } diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs index 39a70f7a1e..e579bb8476 100644 --- a/nexus/db-queries/src/db/datastore/db_metadata.rs +++ b/nexus/db-queries/src/db/datastore/db_metadata.rs @@ -8,10 +8,7 @@ use super::DataStore; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::TransactionError; -use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, -}; +use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use camino::{Utf8Path, Utf8PathBuf}; use chrono::Utc; use diesel::prelude::*; @@ -415,30 +412,30 @@ impl DataStore { target: &SemverVersion, sql: &String, ) -> Result<(), Error> { - let result = self.pool_connection_unauthorized().await?.transaction_async(|conn| async move { - if target.to_string() != EARLIEST_SUPPORTED_VERSION { - let validate_version_query = format!("SELECT CAST(\ - IF(\ - (\ - SELECT version = '{current}' and target_version = '{target}'\ - FROM omicron.public.db_metadata WHERE singleton = true\ - ),\ - 'true',\ - 'Invalid starting version for schema change'\ - ) AS BOOL\ - );"); - conn.batch_execute_async(&validate_version_query).await?; - } - conn.batch_execute_async(&sql).await?; - Ok::<_, TransactionError<()>>(()) - }).await; + let conn = self.pool_connection_unauthorized().await?; + + let result = self.transaction_retry_wrapper("apply_schema_update") + .transaction(&conn, |conn| async move { + if target.to_string() != EARLIEST_SUPPORTED_VERSION { + let validate_version_query = format!("SELECT CAST(\ + IF(\ + (\ + SELECT version = '{current}' and target_version = '{target}'\ + FROM omicron.public.db_metadata WHERE singleton = true\ + ),\ + 'true',\ + 'Invalid starting version for schema change'\ + ) AS BOOL\ + );"); + conn.batch_execute_async(&validate_version_query).await?; + } + conn.batch_execute_async(&sql).await?; + Ok(()) + }).await; match result { Ok(()) => Ok(()), - Err(TransactionError::CustomError(())) => panic!("No custom error"), - Err(TransactionError::Database(e)) => { - Err(public_error_from_diesel(e, ErrorHandler::Server)) - } + Err(e) => Err(public_error_from_diesel(e, ErrorHandler::Server)), } } diff --git a/nexus/db-queries/src/db/datastore/device_auth.rs b/nexus/db-queries/src/db/datastore/device_auth.rs index e1facb43f6..8d8e09744c 100644 --- a/nexus/db-queries/src/db/datastore/device_auth.rs +++ b/nexus/db-queries/src/db/datastore/device_auth.rs @@ -10,10 +10,8 @@ use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::DeviceAccessToken; use crate::db::model::DeviceAuthRequest; -use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use omicron_common::api::external::CreateResult; @@ -75,35 +73,40 @@ impl DataStore { RequestNotFound, TooManyRequests, } - type TxnError = TransactionError; - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - match delete_request.execute_async(&conn).await? { - 0 => { - Err(TxnError::CustomError(TokenGrantError::RequestNotFound)) + let err = crate::transaction_retry::OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("device_access_token_create") + .transaction(&conn, |conn| { + let err = err.clone(); + let insert_token = insert_token.clone(); + let delete_request = delete_request.clone(); + async move { + match delete_request.execute_async(&conn).await? { + 0 => Err(err.bail(TokenGrantError::RequestNotFound)), + 1 => Ok(insert_token.get_result_async(&conn).await?), + _ => Err(err.bail(TokenGrantError::TooManyRequests)), } - 1 => Ok(insert_token.get_result_async(&conn).await?), - _ => Err(TxnError::CustomError( - TokenGrantError::TooManyRequests, - )), } }) .await - .map_err(|e| match e { - TxnError::CustomError(TokenGrantError::RequestNotFound) => { - Error::ObjectNotFound { - type_name: ResourceType::DeviceAuthRequest, - lookup_type: LookupType::ByCompositeId( - authz_request.id(), - ), + .map_err(|e| { + if let Some(err) = err.take() { + match err { + TokenGrantError::RequestNotFound => { + Error::ObjectNotFound { + type_name: ResourceType::DeviceAuthRequest, + lookup_type: LookupType::ByCompositeId( + authz_request.id(), + ), + } + } + TokenGrantError::TooManyRequests => { + Error::internal_error("unexpectedly found multiple device auth requests for the same user code") + } } - } - TxnError::CustomError(TokenGrantError::TooManyRequests) => { - Error::internal_error("unexpectedly found multiple device auth requests for the same user code") - } - TxnError::Database(e) => { + } else { public_error_from_diesel(e, ErrorHandler::Server) } }) diff --git a/nexus/db-queries/src/db/datastore/dns.rs b/nexus/db-queries/src/db/datastore/dns.rs index f7ad97593e..552ad31487 100644 --- a/nexus/db-queries/src/db/datastore/dns.rs +++ b/nexus/db-queries/src/db/datastore/dns.rs @@ -15,6 +15,7 @@ use crate::db::model::DnsZone; use crate::db::model::Generation; use crate::db::model::InitialDnsGroup; use crate::db::pagination::paginated; +use crate::db::pagination::Paginator; use crate::db::pool::DbConnection; use crate::db::TransactionError; use async_bb8_diesel::AsyncConnection; @@ -67,7 +68,9 @@ impl DataStore { dns_group: DnsGroup, ) -> ListResultVec { let conn = self.pool_connection_authorized(opctx).await?; - self.dns_zones_list_all_on_connection(opctx, &conn, dns_group).await + Ok(self + .dns_zones_list_all_on_connection(opctx, &conn, dns_group) + .await?) } /// Variant of [`Self::dns_zones_list_all`] which may be called from a @@ -77,7 +80,7 @@ impl DataStore { opctx: &OpContext, conn: &async_bb8_diesel::Connection, dns_group: DnsGroup, - ) -> ListResultVec { + ) -> Result, TransactionError> { use db::schema::dns_zone::dsl; const LIMIT: usize = 5; @@ -88,8 +91,7 @@ impl DataStore { .limit(i64::try_from(LIMIT).unwrap()) .select(DnsZone::as_select()) .load_async(conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .await?; bail_unless!( list.len() < LIMIT, @@ -106,12 +108,14 @@ impl DataStore { opctx: &OpContext, dns_group: DnsGroup, ) -> LookupResult { - self.dns_group_latest_version_conn( - opctx, - &*self.pool_connection_authorized(opctx).await?, - dns_group, - ) - .await + let version = self + .dns_group_latest_version_conn( + opctx, + &*self.pool_connection_authorized(opctx).await?, + dns_group, + ) + .await?; + Ok(version) } pub async fn dns_group_latest_version_conn( @@ -119,7 +123,7 @@ impl DataStore { opctx: &OpContext, conn: &async_bb8_diesel::Connection, dns_group: DnsGroup, - ) -> LookupResult { + ) -> Result> { opctx.authorize(authz::Action::Read, &authz::DNS_CONFIG).await?; use db::schema::dns_version::dsl; let versions = dsl::dns_version @@ -128,8 +132,7 @@ impl DataStore { .limit(1) .select(DnsVersion::as_select()) .load_async(conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .await?; bail_unless!( versions.len() == 1, @@ -240,9 +243,8 @@ impl DataStore { let mut zones = Vec::with_capacity(dns_zones.len()); for zone in dns_zones { let mut zone_records = Vec::new(); - let mut marker = None; - - loop { + let mut paginator = Paginator::new(batch_size); + while let Some(p) = paginator.next() { debug!(log, "listing DNS names for zone"; "dns_zone_id" => zone.id.to_string(), "dns_zone_name" => &zone.zone_name, @@ -250,25 +252,16 @@ impl DataStore { "found_so_far" => zone_records.len(), "batch_size" => batch_size.get(), ); - let pagparams = DataPageParams { - marker: marker.as_ref(), - direction: dropshot::PaginationOrder::Ascending, - limit: batch_size, - }; let names_batch = self - .dns_names_list(opctx, zone.id, version.version, &pagparams) + .dns_names_list( + opctx, + zone.id, + version.version, + &p.current_pagparams(), + ) .await?; - let done = names_batch.len() - < usize::try_from(batch_size.get()).unwrap(); - if let Some((last_name, _)) = names_batch.last() { - marker = Some(last_name.clone()); - } else { - assert!(done); - } + paginator = p.found_batch(&names_batch, &|(n, _)| n.clone()); zone_records.extend(names_batch.into_iter()); - if done { - break; - } } debug!(log, "found all DNS names for zone"; @@ -377,28 +370,17 @@ impl DataStore { opctx: &OpContext, conn: &async_bb8_diesel::Connection, update: DnsVersionUpdateBuilder, - ) -> Result<(), Error> { + ) -> Result<(), TransactionError> { opctx.authorize(authz::Action::Modify, &authz::DNS_CONFIG).await?; let zones = self .dns_zones_list_all_on_connection(opctx, conn, update.dns_group) .await?; - let result = conn - .transaction_async(|c| async move { - self.dns_update_internal(opctx, &c, update, zones) - .await - .map_err(TransactionError::CustomError) - }) - .await; - - match result { - Ok(()) => Ok(()), - Err(TransactionError::CustomError(e)) => Err(e), - Err(TransactionError::Database(e)) => { - Err(public_error_from_diesel(e, ErrorHandler::Server)) - } - } + conn.transaction_async(|c| async move { + self.dns_update_internal(opctx, &c, update, zones).await + }) + .await } // This must only be used inside a transaction. Otherwise, it may make @@ -409,7 +391,7 @@ impl DataStore { conn: &async_bb8_diesel::Connection, update: DnsVersionUpdateBuilder, zones: Vec, - ) -> Result<(), Error> { + ) -> Result<(), TransactionError> { // TODO-scalability TODO-performance This would be much better as a CTE // for all the usual reasons described in RFD 192. Using an interactive // transaction here means that either we wind up holding database locks @@ -455,10 +437,7 @@ impl DataStore { diesel::insert_into(dsl::dns_version) .values(new_version) .execute_async(conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - })?; + .await?; } { @@ -480,8 +459,7 @@ impl DataStore { ) .set(dsl::version_removed.eq(new_version_num)) .execute_async(conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .await?; bail_unless!( nremoved == ntoremove, @@ -495,10 +473,7 @@ impl DataStore { let nadded = diesel::insert_into(dsl::dns_name) .values(new_names) .execute_async(conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - })?; + .await?; bail_unless!( nadded == ntoadd, @@ -1684,6 +1659,10 @@ mod test { let conn = datastore.pool_connection_for_tests().await.unwrap(); let error = datastore.dns_update(&opctx, &conn, update).await.unwrap_err(); + let error = match error { + TransactionError::CustomError(err) => err, + _ => panic!("Unexpected error: {:?}", error), + }; assert_eq!( error.to_string(), "Internal Error: updated wrong number of dns_name \ @@ -1707,11 +1686,15 @@ mod test { update.add_name(String::from("n2"), records1.clone()).unwrap(); let conn = datastore.pool_connection_for_tests().await.unwrap(); - let error = - datastore.dns_update(&opctx, &conn, update).await.unwrap_err(); + let error = Error::from( + datastore.dns_update(&opctx, &conn, update).await.unwrap_err(), + ); let msg = error.to_string(); - assert!(msg.starts_with("Internal Error: ")); - assert!(msg.contains("violates unique constraint")); + assert!(msg.starts_with("Internal Error: "), "Message: {msg:}"); + assert!( + msg.contains("violates unique constraint"), + "Message: {msg:}" + ); } let dns_config = datastore diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs index e663130a84..e821082501 100644 --- a/nexus/db-queries/src/db/datastore/external_ip.rs +++ b/nexus/db-queries/src/db/datastore/external_ip.rs @@ -10,12 +10,16 @@ use crate::authz::ApiResource; use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; +use crate::db::error::TransactionError; use crate::db::lookup::LookupPath; use crate::db::model::ExternalIp; +use crate::db::model::FloatingIp; use crate::db::model::IncompleteExternalIp; use crate::db::model::IpKind; use crate::db::model::Name; +use crate::db::pagination::paginated; use crate::db::pool::DbConnection; use crate::db::queries::external_ip::NextExternalIp; use crate::db::update_and_check::UpdateAndCheck; @@ -23,10 +27,18 @@ use crate::db::update_and_check::UpdateStatus; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; +use nexus_types::external_api::params; use nexus_types::identity::Resource; +use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; +use omicron_common::api::external::NameOrId; +use omicron_common::api::external::ResourceType; +use omicron_common::api::external::UpdateResult; +use ref_cast::RefCast; use std::net::IpAddr; use uuid::Uuid; @@ -126,13 +138,64 @@ impl DataStore { self.allocate_external_ip(opctx, data).await } + /// Allocates a floating IP address for instance usage. + pub async fn allocate_floating_ip( + &self, + opctx: &OpContext, + project_id: Uuid, + params: params::FloatingIpCreate, + ) -> CreateResult { + let ip_id = Uuid::new_v4(); + + let pool_id = match params.pool { + Some(NameOrId::Name(name)) => { + LookupPath::new(opctx, self) + .ip_pool_name(&Name(name)) + .fetch_for(authz::Action::Read) + .await? + .1 + } + Some(NameOrId::Id(id)) => { + LookupPath::new(opctx, self) + .ip_pool_id(id) + .fetch_for(authz::Action::Read) + .await? + .1 + } + None => self.ip_pools_fetch_default(opctx).await?, + } + .id(); + + let data = if let Some(ip) = params.address { + IncompleteExternalIp::for_floating_explicit( + ip_id, + &Name(params.identity.name), + ¶ms.identity.description, + project_id, + ip, + pool_id, + ) + } else { + IncompleteExternalIp::for_floating( + ip_id, + &Name(params.identity.name), + ¶ms.identity.description, + project_id, + pool_id, + ) + }; + + self.allocate_external_ip(opctx, data).await + } + async fn allocate_external_ip( &self, opctx: &OpContext, data: IncompleteExternalIp, ) -> CreateResult { let conn = self.pool_connection_authorized(opctx).await?; - Self::allocate_external_ip_on_connection(&conn, data).await + let ip = Self::allocate_external_ip_on_connection(&conn, data).await?; + Ok(ip) } /// Variant of [Self::allocate_external_ip] which may be called from a @@ -140,23 +203,46 @@ impl DataStore { pub(crate) async fn allocate_external_ip_on_connection( conn: &async_bb8_diesel::Connection, data: IncompleteExternalIp, - ) -> CreateResult { + ) -> Result> { + use diesel::result::DatabaseErrorKind::UniqueViolation; + // Name needs to be cloned out here (if present) to give users a + // sensible error message on name collision. + let name = data.name().clone(); let explicit_ip = data.explicit_ip().is_some(); NextExternalIp::new(data).get_result_async(conn).await.map_err(|e| { + use diesel::result::Error::DatabaseError; use diesel::result::Error::NotFound; match e { NotFound => { if explicit_ip { - Error::invalid_request( + TransactionError::CustomError(Error::invalid_request( "Requested external IP address not available", - ) + )) } else { - Error::invalid_request( + TransactionError::CustomError(Error::invalid_request( "No external IP addresses available", - ) + )) + } + } + DatabaseError(UniqueViolation, ..) if name.is_some() => { + TransactionError::CustomError(public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::FloatingIp, + name.as_ref() + .map(|m| m.as_str()) + .unwrap_or_default(), + ), + )) + } + _ => { + if retryable(&e) { + return TransactionError::Database(e); } + TransactionError::CustomError( + crate::db::queries::external_ip::from_diesel(e), + ) } - _ => crate::db::queries::external_ip::from_diesel(e), } }) } @@ -245,8 +331,6 @@ impl DataStore { /// This method returns the number of records deleted, rather than the usual /// `DeleteResult`. That's mostly useful for tests, but could be important /// if callers have some invariants they'd like to check. - // TODO-correctness: This can't be used for Floating IPs, we'll need a - // _detatch_ method for that. pub async fn deallocate_external_ip_by_instance_id( &self, opctx: &OpContext, @@ -265,6 +349,27 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + /// Detach an individual Floating IP address from its parent instance. + /// + /// As in `deallocate_external_ip_by_instance_id`, this method returns the + /// number of records altered, rather than an `UpdateResult`. + pub async fn detach_floating_ips_by_instance_id( + &self, + opctx: &OpContext, + instance_id: Uuid, + ) -> Result { + use db::schema::external_ip::dsl; + diesel::update(dsl::external_ip) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::is_service.eq(false)) + .filter(dsl::parent_id.eq(instance_id)) + .filter(dsl::kind.eq(IpKind::Floating)) + .set(dsl::parent_id.eq(Option::::None)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + /// Fetch all external IP addresses of any kind for the provided instance pub async fn instance_lookup_external_ips( &self, @@ -281,4 +386,167 @@ impl DataStore { .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + + /// Fetch all Floating IP addresses for the provided project. + pub async fn floating_ips_list( + &self, + opctx: &OpContext, + authz_project: &authz::Project, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::floating_ip::dsl; + + opctx.authorize(authz::Action::ListChildren, authz_project).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::floating_ip, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::floating_ip, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::project_id.eq(authz_project.id())) + .filter(dsl::time_deleted.is_null()) + .select(FloatingIp::as_select()) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + /// Delete a Floating IP, verifying first that it is not in use. + pub async fn floating_ip_delete( + &self, + opctx: &OpContext, + authz_fip: &authz::FloatingIp, + db_fip: &FloatingIp, + ) -> DeleteResult { + use db::schema::external_ip::dsl; + + // Verify this FIP is not attached to any instances/services. + if db_fip.parent_id.is_some() { + return Err(Error::invalid_request( + "Floating IP cannot be deleted while attached to an instance", + )); + } + + opctx.authorize(authz::Action::Delete, authz_fip).await?; + + let now = Utc::now(); + let updated_rows = diesel::update(dsl::external_ip) + .filter(dsl::id.eq(db_fip.id())) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::parent_id.is_null()) + .set(dsl::time_deleted.eq(now)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_fip), + ) + })?; + + if updated_rows == 0 { + return Err(Error::InvalidRequest { + message: "deletion failed due to concurrent modification" + .to_string(), + }); + } + Ok(()) + } + + /// Attaches a Floating IP address to an instance. + pub async fn floating_ip_attach( + &self, + opctx: &OpContext, + authz_fip: &authz::FloatingIp, + db_fip: &FloatingIp, + instance_id: Uuid, + ) -> UpdateResult { + use db::schema::external_ip::dsl; + + // Verify this FIP is not attached to any instances/services. + if db_fip.parent_id.is_some() { + return Err(Error::invalid_request( + "Floating IP cannot be attached to one instance while still attached to another", + )); + } + + let (.., authz_instance, _db_instance) = LookupPath::new(&opctx, self) + .instance_id(instance_id) + .fetch_for(authz::Action::Modify) + .await?; + + opctx.authorize(authz::Action::Modify, authz_fip).await?; + opctx.authorize(authz::Action::Modify, &authz_instance).await?; + + diesel::update(dsl::external_ip) + .filter(dsl::id.eq(db_fip.id())) + .filter(dsl::kind.eq(IpKind::Floating)) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::parent_id.is_null()) + .set(( + dsl::parent_id.eq(Some(instance_id)), + dsl::time_modified.eq(Utc::now()), + )) + .returning(ExternalIp::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_fip), + ) + }) + .and_then(|r| FloatingIp::try_from(r)) + .map_err(|e| Error::internal_error(&format!("{e}"))) + } + + /// Detaches a Floating IP address from an instance. + pub async fn floating_ip_detach( + &self, + opctx: &OpContext, + authz_fip: &authz::FloatingIp, + db_fip: &FloatingIp, + ) -> UpdateResult { + use db::schema::external_ip::dsl; + + let Some(instance_id) = db_fip.parent_id else { + return Err(Error::invalid_request( + "Floating IP is not attached to an instance", + )); + }; + + let (.., authz_instance, _db_instance) = LookupPath::new(&opctx, self) + .instance_id(instance_id) + .fetch_for(authz::Action::Modify) + .await?; + + opctx.authorize(authz::Action::Modify, authz_fip).await?; + opctx.authorize(authz::Action::Modify, &authz_instance).await?; + + diesel::update(dsl::external_ip) + .filter(dsl::id.eq(db_fip.id())) + .filter(dsl::kind.eq(IpKind::Floating)) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::parent_id.eq(instance_id)) + .set(( + dsl::parent_id.eq(Option::::None), + dsl::time_modified.eq(Utc::now()), + )) + .returning(ExternalIp::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_fip), + ) + }) + .and_then(|r| FloatingIp::try_from(r)) + .map_err(|e| Error::internal_error(&format!("{e}"))) + } } diff --git a/nexus/db-queries/src/db/datastore/identity_provider.rs b/nexus/db-queries/src/db/datastore/identity_provider.rs index fdc9a020e7..cee577acd6 100644 --- a/nexus/db-queries/src/db/datastore/identity_provider.rs +++ b/nexus/db-queries/src/db/datastore/identity_provider.rs @@ -14,7 +14,6 @@ use crate::db::identity::Resource; use crate::db::model::IdentityProvider; use crate::db::model::Name; use crate::db::pagination::paginated; -use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use omicron_common::api::external::http_pagination::PaginatedBy; @@ -63,36 +62,47 @@ impl DataStore { assert_eq!(provider.silo_id, authz_idp_list.silo().id()); let name = provider.identity().name.to_string(); - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - // insert silo identity provider record with type Saml - use db::schema::identity_provider::dsl as idp_dsl; - diesel::insert_into(idp_dsl::identity_provider) - .values(db::model::IdentityProvider { - identity: db::model::IdentityProviderIdentity { - id: provider.identity.id, - name: provider.identity.name.clone(), - description: provider.identity.description.clone(), - time_created: provider.identity.time_created, - time_modified: provider.identity.time_modified, - time_deleted: provider.identity.time_deleted, - }, - silo_id: provider.silo_id, - provider_type: db::model::IdentityProviderType::Saml, - }) - .execute_async(&conn) - .await?; + let conn = self.pool_connection_authorized(opctx).await?; - // insert silo saml identity provider record - use db::schema::saml_identity_provider::dsl; - let result = diesel::insert_into(dsl::saml_identity_provider) - .values(provider) - .returning(db::model::SamlIdentityProvider::as_returning()) - .get_result_async(&conn) - .await?; + self.transaction_retry_wrapper("saml_identity_provider_create") + .transaction(&conn, |conn| { + let provider = provider.clone(); + async move { + // insert silo identity provider record with type Saml + use db::schema::identity_provider::dsl as idp_dsl; + diesel::insert_into(idp_dsl::identity_provider) + .values(db::model::IdentityProvider { + identity: db::model::IdentityProviderIdentity { + id: provider.identity.id, + name: provider.identity.name.clone(), + description: provider + .identity + .description + .clone(), + time_created: provider.identity.time_created, + time_modified: provider.identity.time_modified, + time_deleted: provider.identity.time_deleted, + }, + silo_id: provider.silo_id, + provider_type: + db::model::IdentityProviderType::Saml, + }) + .execute_async(&conn) + .await?; - Ok(result) + // insert silo saml identity provider record + use db::schema::saml_identity_provider::dsl; + let result = + diesel::insert_into(dsl::saml_identity_provider) + .values(provider) + .returning( + db::model::SamlIdentityProvider::as_returning(), + ) + .get_result_async(&conn) + .await?; + + Ok(result) + } }) .await .map_err(|e| { diff --git a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs index 274937b299..1caf5617bb 100644 --- a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs +++ b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs @@ -36,6 +36,7 @@ impl DataStore { .filter(dsl::sled_address.eq(nat_entry.sled_address)) .filter(dsl::vni.eq(nat_entry.vni)) .filter(dsl::mac.eq(nat_entry.mac)) + .filter(dsl::version_removed.is_null()) .select(( dsl::external_address, dsl::first_port, @@ -275,7 +276,7 @@ mod test { use crate::db::datastore::datastore_test; use chrono::Utc; - use nexus_db_model::{Ipv4NatValues, MacAddr, Vni}; + use nexus_db_model::{Ipv4NatEntry, Ipv4NatValues, MacAddr, Vni}; use nexus_test_utils::db::test_setup_database; use omicron_common::api::external; use omicron_test_utils::dev; @@ -427,7 +428,6 @@ mod test { datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); assert_eq!(nat_entries.len(), 1); - // version should be unchanged assert_eq!( datastore.ipv4_nat_current_version(&opctx).await.unwrap(), @@ -437,4 +437,150 @@ mod test { db.cleanup().await.unwrap(); logctx.cleanup_successful(); } + + #[tokio::test] + /// Table design and queries should only insert one active NAT entry for a given + /// set of properties, but allow multiple deleted nat entries for the same set + /// of properties. + async fn table_allows_unique_active_multiple_deleted() { + let logctx = dev::test_setup_log("test_nat_version_tracking"); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // We should not have any NAT entries at this moment + let initial_state = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert!(initial_state.is_empty()); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 0 + ); + + // Each change (creation / deletion) to the NAT table should increment the + // version number of the row in the NAT table + let external_address = external::Ipv4Net( + ipnetwork::Ipv4Network::try_from("10.0.0.100").unwrap(), + ); + + let sled_address = external::Ipv6Net( + ipnetwork::Ipv6Network::try_from("fd00:1122:3344:104::1").unwrap(), + ); + + // Add a nat entry. + let nat1 = Ipv4NatValues { + external_address: external_address.into(), + first_port: 0.into(), + last_port: 999.into(), + sled_address: sled_address.into(), + vni: Vni(external::Vni::random()), + mac: MacAddr( + external::MacAddr::from_str("A8:40:25:F5:EB:2A").unwrap(), + ), + }; + + datastore.ensure_ipv4_nat_entry(&opctx, nat1.clone()).await.unwrap(); + + // Try to add it again. It should still only result in a single entry. + datastore.ensure_ipv4_nat_entry(&opctx, nat1.clone()).await.unwrap(); + let first_entry = datastore + .ipv4_nat_find_by_values(&opctx, nat1.clone()) + .await + .unwrap(); + + let nat_entries = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + // The NAT table has undergone one change. One entry has been added, + // none deleted, so we should be at version 1. + assert_eq!(nat_entries.len(), 1); + assert_eq!(nat_entries.last().unwrap().version_added, 1); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 1 + ); + + datastore.ipv4_nat_delete(&opctx, &first_entry).await.unwrap(); + + // The NAT table has undergone two changes. One entry has been added, + // then deleted, so we should be at version 2. + let nat_entries = datastore + .ipv4_nat_list_since_version(&opctx, 0, 10) + .await + .unwrap() + .into_iter(); + + let active: Vec = nat_entries + .clone() + .filter(|entry| entry.version_removed.is_none()) + .collect(); + + let inactive: Vec = nat_entries + .filter(|entry| entry.version_removed.is_some()) + .collect(); + + assert!(active.is_empty()); + assert_eq!(inactive.len(), 1); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 2 + ); + + // Add the same entry back. This simulates the behavior we will see + // when stopping and then restarting an instance. + datastore.ensure_ipv4_nat_entry(&opctx, nat1.clone()).await.unwrap(); + + // The NAT table has undergone three changes. + let nat_entries = datastore + .ipv4_nat_list_since_version(&opctx, 0, 10) + .await + .unwrap() + .into_iter(); + + let active: Vec = nat_entries + .clone() + .filter(|entry| entry.version_removed.is_none()) + .collect(); + + let inactive: Vec = nat_entries + .filter(|entry| entry.version_removed.is_some()) + .collect(); + + assert_eq!(active.len(), 1); + assert_eq!(inactive.len(), 1); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 3 + ); + + let second_entry = + datastore.ipv4_nat_find_by_values(&opctx, nat1).await.unwrap(); + datastore.ipv4_nat_delete(&opctx, &second_entry).await.unwrap(); + + // The NAT table has undergone four changes + let nat_entries = datastore + .ipv4_nat_list_since_version(&opctx, 0, 10) + .await + .unwrap() + .into_iter(); + + let active: Vec = nat_entries + .clone() + .filter(|entry| entry.version_removed.is_none()) + .collect(); + + let inactive: Vec = nat_entries + .filter(|entry| entry.version_removed.is_some()) + .collect(); + + assert_eq!(active.len(), 0); + assert_eq!(inactive.len(), 2); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 4 + ); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } } diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 44cd7a95b7..2844285f40 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -148,6 +148,7 @@ pub type DataStoreConnection<'a> = pub struct DataStore { pool: Arc, virtual_provisioning_collection_producer: crate::provisioning::Producer, + transaction_retry_producer: crate::transaction_retry::Producer, } // The majority of `DataStore`'s methods live in our submodules as a concession @@ -164,6 +165,8 @@ impl DataStore { pool, virtual_provisioning_collection_producer: crate::provisioning::Producer::new(), + transaction_retry_producer: crate::transaction_retry::Producer::new( + ), }; Ok(datastore) } @@ -210,6 +213,29 @@ impl DataStore { self.virtual_provisioning_collection_producer.clone(), ) .unwrap(); + registry + .register_producer(self.transaction_retry_producer.clone()) + .unwrap(); + } + + /// Constructs a transaction retry helper + /// + /// Automatically wraps the underlying producer + pub fn transaction_retry_wrapper( + &self, + name: &'static str, + ) -> crate::transaction_retry::RetryHelper { + crate::transaction_retry::RetryHelper::new( + &self.transaction_retry_producer, + name, + ) + } + + #[cfg(test)] + pub(crate) fn transaction_retry_producer( + &self, + ) -> &crate::transaction_retry::Producer { + &self.transaction_retry_producer } /// Returns a connection to a connection from the database connection pool. @@ -1635,6 +1661,7 @@ mod test { time_deleted: None, ip_pool_id: Uuid::new_v4(), ip_pool_range_id: Uuid::new_v4(), + project_id: None, is_service: false, parent_id: Some(instance_id), kind: IpKind::Ephemeral, @@ -1695,6 +1722,7 @@ mod test { time_deleted: None, ip_pool_id: Uuid::new_v4(), ip_pool_range_id: Uuid::new_v4(), + project_id: None, is_service: false, parent_id: Some(Uuid::new_v4()), kind: IpKind::SNat, @@ -1741,6 +1769,7 @@ mod test { use crate::db::model::IpKind; use crate::db::schema::external_ip::dsl; use diesel::result::DatabaseErrorKind::CheckViolation; + use diesel::result::DatabaseErrorKind::UniqueViolation; use diesel::result::Error::DatabaseError; let logctx = dev::test_setup_log("test_external_ip_check_constraints"); @@ -1765,6 +1794,7 @@ mod test { time_deleted: None, ip_pool_id: Uuid::new_v4(), ip_pool_range_id: Uuid::new_v4(), + project_id: None, is_service: false, parent_id: Some(Uuid::new_v4()), kind: IpKind::Floating, @@ -1777,151 +1807,190 @@ mod test { // - name // - description // - parent (instance / service) UUID - let names = [ - None, - Some(db::model::Name(Name::try_from("foo".to_string()).unwrap())), - ]; + // - project UUID + let names = [None, Some("foo")]; let descriptions = [None, Some("foo".to_string())]; let parent_ids = [None, Some(Uuid::new_v4())]; + let project_ids = [None, Some(Uuid::new_v4())]; + + let mut seen_pairs = HashSet::new(); // For Floating IPs, both name and description must be non-NULL - for name in names.iter() { - for description in descriptions.iter() { - for parent_id in parent_ids.iter() { - for is_service in [false, true] { - let new_ip = ExternalIp { - id: Uuid::new_v4(), - name: name.clone(), - description: description.clone(), - ip: addresses.next().unwrap().into(), - is_service, - parent_id: *parent_id, - ..ip - }; - let res = diesel::insert_into(dsl::external_ip) - .values(new_ip) - .execute_async(&*conn) - .await; - if name.is_some() && description.is_some() { - // Name/description must be non-NULL, instance ID can be - // either - res.unwrap_or_else(|_| { - panic!( - "Failed to insert Floating IP with valid \ - name, description, and {} ID", - if is_service { - "Service" - } else { - "Instance" - } - ) - }); - } else { - // At least one is not valid, we expect a check violation - let err = res.expect_err( - "Expected a CHECK violation when inserting a \ - Floating IP record with NULL name and/or description", - ); - assert!( - matches!( - err, - DatabaseError( - CheckViolation, - _ - ) - ), - "Expected a CHECK violation when inserting a \ - Floating IP record with NULL name and/or description", - ); - } - } - } + // If they are instance FIPs, they *must* have a project id. + for ( + name, + description, + parent_id, + is_service, + project_id, + modify_name, + ) in itertools::iproduct!( + &names, + &descriptions, + &parent_ids, + [false, true], + &project_ids, + [false, true] + ) { + // Both choices of parent_id are valid, so we need a unique name for each. + let name_local = name.map(|v| { + let name = if modify_name { + v.to_string() + } else { + format!("{v}-with-parent") + }; + db::model::Name(Name::try_from(name).unwrap()) + }); + + // We do name duplicate checking on the `Some` branch, don't steal the + // name intended for another floating IP. + if parent_id.is_none() && modify_name { + continue; + } + + let new_ip = ExternalIp { + id: Uuid::new_v4(), + name: name_local.clone(), + description: description.clone(), + ip: addresses.next().unwrap().into(), + is_service, + parent_id: *parent_id, + project_id: *project_id, + ..ip + }; + + let key = (*project_id, name_local); + + let res = diesel::insert_into(dsl::external_ip) + .values(new_ip) + .execute_async(&*conn) + .await; + + let project_as_expected = (is_service && project_id.is_none()) + || (!is_service && project_id.is_some()); + + let valid_expression = + name.is_some() && description.is_some() && project_as_expected; + let name_exists = seen_pairs.contains(&key); + + if valid_expression && !name_exists { + // Name/description must be non-NULL, instance ID can be + // either + // Names must be unique at fleet level and at project level. + // Project must be NULL if service, non-NULL if instance. + res.unwrap_or_else(|e| { + panic!( + "Failed to insert Floating IP with valid \ + name, description, project ID, and {} ID:\ + {name:?} {description:?} {project_id:?} {:?}\n{e}", + if is_service { "Service" } else { "Instance" }, + &ip.parent_id + ) + }); + + seen_pairs.insert(key); + } else if !valid_expression { + // Several permutations are invalid and we want to detect them all. + // NOTE: CHECK violation will supersede UNIQUE violation below. + let err = res.expect_err( + "Expected a CHECK violation when inserting a \ + Floating IP record with NULL name and/or description, \ + and incorrect project parent relation", + ); + assert!( + matches!(err, DatabaseError(CheckViolation, _)), + "Expected a CHECK violation when inserting a \ + Floating IP record with NULL name and/or description, \ + and incorrect project parent relation", + ); + } else { + let err = res.expect_err( + "Expected a UNIQUE violation when inserting a \ + Floating IP record with existing (name, project_id)", + ); + assert!( + matches!(err, DatabaseError(UniqueViolation, _)), + "Expected a UNIQUE violation when inserting a \ + Floating IP record with existing (name, project_id)", + ); } } - // For other IP types, both name and description must be NULL - for kind in [IpKind::SNat, IpKind::Ephemeral].into_iter() { - for name in names.iter() { - for description in descriptions.iter() { - for parent_id in parent_ids.iter() { - for is_service in [false, true] { - let new_ip = ExternalIp { - id: Uuid::new_v4(), - name: name.clone(), - description: description.clone(), - kind, - ip: addresses.next().unwrap().into(), - is_service, - parent_id: *parent_id, - ..ip - }; - let res = diesel::insert_into(dsl::external_ip) - .values(new_ip.clone()) - .execute_async(&*conn) - .await; - let ip_type = - if is_service { "Service" } else { "Instance" }; - if name.is_none() - && description.is_none() - && parent_id.is_some() - { - // Name/description must be NULL, instance ID cannot - // be NULL. - - if kind == IpKind::Ephemeral && is_service { - // Ephemeral Service IPs aren't supported. - let err = res.unwrap_err(); - assert!( - matches!( - err, - DatabaseError( - CheckViolation, - _ - ) - ), - "Expected a CHECK violation when inserting an \ - Ephemeral Service IP", - ); - } else { - assert!( - res.is_ok(), - "Failed to insert {:?} IP with valid \ - name, description, and {} ID", - kind, - ip_type, - ); - } - } else { - // One is not valid, we expect a check violation - assert!( - res.is_err(), - "Expected a CHECK violation when inserting a \ - {:?} IP record with non-NULL name, description, \ - and/or {} ID", - kind, - ip_type, - ); - let err = res.unwrap_err(); - assert!( - matches!( - err, - DatabaseError( - CheckViolation, - _ - ) - ), - "Expected a CHECK violation when inserting a \ - {:?} IP record with non-NULL name, description, \ - and/or {} ID", - kind, - ip_type, - ); - } - } - } + // For other IP types: name, description and project must be NULL + for (kind, name, description, parent_id, is_service, project_id) in itertools::iproduct!( + [IpKind::SNat, IpKind::Ephemeral], + &names, + &descriptions, + &parent_ids, + [false, true], + &project_ids + ) { + let name_local = name.map(|v| { + db::model::Name(Name::try_from(v.to_string()).unwrap()) + }); + let new_ip = ExternalIp { + id: Uuid::new_v4(), + name: name_local, + description: description.clone(), + kind, + ip: addresses.next().unwrap().into(), + is_service, + parent_id: *parent_id, + project_id: *project_id, + ..ip + }; + let res = diesel::insert_into(dsl::external_ip) + .values(new_ip.clone()) + .execute_async(&*conn) + .await; + let ip_type = if is_service { "Service" } else { "Instance" }; + if name.is_none() + && description.is_none() + && parent_id.is_some() + && project_id.is_none() + { + // Name/description must be NULL, instance ID cannot + // be NULL. + + if kind == IpKind::Ephemeral && is_service { + // Ephemeral Service IPs aren't supported. + let err = res.unwrap_err(); + assert!( + matches!(err, DatabaseError(CheckViolation, _)), + "Expected a CHECK violation when inserting an \ + Ephemeral Service IP", + ); + } else { + assert!( + res.is_ok(), + "Failed to insert {:?} IP with valid \ + name, description, and {} ID", + kind, + ip_type, + ); } + } else { + // One is not valid, we expect a check violation + assert!( + res.is_err(), + "Expected a CHECK violation when inserting a \ + {:?} IP record with non-NULL name, description, \ + and/or {} ID", + kind, + ip_type, + ); + let err = res.unwrap_err(); + assert!( + matches!(err, DatabaseError(CheckViolation, _)), + "Expected a CHECK violation when inserting a \ + {:?} IP record with non-NULL name, description, \ + and/or {} ID", + kind, + ip_type, + ); } } + db.cleanup().await.unwrap(); logctx.cleanup_successful(); } diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 06550e9439..4d4e43c9a7 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -13,7 +13,6 @@ use crate::db::collection_insert::DatastoreCollection; use crate::db::cte_utils::BoxedQuery; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::IncompleteNetworkInterface; use crate::db::model::Instance; use crate::db::model::InstanceNetworkInterface; @@ -25,7 +24,7 @@ use crate::db::model::VpcSubnet; use crate::db::pagination::paginated; use crate::db::pool::DbConnection; use crate::db::queries::network_interface; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; @@ -466,77 +465,91 @@ impl DataStore { InstanceNotStopped, FailedToUnsetPrimary(DieselError), } - type TxnError = TransactionError; + + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; if primary { - conn.transaction_async(|conn| async move { - let instance_runtime = - instance_query.get_result_async(&conn).await?.runtime_state; - if instance_runtime.propolis_id.is_some() - || instance_runtime.nexus_state != stopped - { - return Err(TxnError::CustomError( - NetworkInterfaceUpdateError::InstanceNotStopped, - )); - } + self.transaction_retry_wrapper("instance_update_network_interface") + .transaction(&conn, |conn| { + let err = err.clone(); + let stopped = stopped.clone(); + let update_target_query = update_target_query.clone(); + async move { + let instance_runtime = + instance_query.get_result_async(&conn).await?.runtime_state; + if instance_runtime.propolis_id.is_some() + || instance_runtime.nexus_state != stopped + { + return Err(err.bail(NetworkInterfaceUpdateError::InstanceNotStopped)); + } - // First, get the primary interface - let primary_interface = - find_primary_query.get_result_async(&conn).await?; - // If the target and primary are different, we need to toggle - // the primary into a secondary. - if primary_interface.identity.id != interface_id { - use crate::db::schema::network_interface::dsl; - if let Err(e) = diesel::update(dsl::network_interface) - .filter(dsl::id.eq(primary_interface.identity.id)) - .filter(dsl::kind.eq(NetworkInterfaceKind::Instance)) - .filter(dsl::time_deleted.is_null()) - .set(dsl::is_primary.eq(false)) - .execute_async(&conn) - .await - { - return Err(TxnError::CustomError( - NetworkInterfaceUpdateError::FailedToUnsetPrimary( - e, - ), - )); - } - } + // First, get the primary interface + let primary_interface = + find_primary_query.get_result_async(&conn).await?; + // If the target and primary are different, we need to toggle + // the primary into a secondary. + if primary_interface.identity.id != interface_id { + use crate::db::schema::network_interface::dsl; + if let Err(e) = diesel::update(dsl::network_interface) + .filter(dsl::id.eq(primary_interface.identity.id)) + .filter(dsl::kind.eq(NetworkInterfaceKind::Instance)) + .filter(dsl::time_deleted.is_null()) + .set(dsl::is_primary.eq(false)) + .execute_async(&conn) + .await + { + return Err(err.bail_retryable_or_else( + e, + |e| NetworkInterfaceUpdateError::FailedToUnsetPrimary(e) + )); + } + } - // In any case, update the actual target - Ok(update_target_query.get_result_async(&conn).await?) - }) + // In any case, update the actual target + update_target_query.get_result_async(&conn).await + } + }).await } else { // In this case, we can just directly apply the updates. By // construction, `updates.primary` is `None`, so nothing will // be done there. The other columns always need to be updated, and // we're only hitting a single row. Note that we still need to // verify the instance is stopped. - conn.transaction_async(|conn| async move { - let instance_state = - instance_query.get_result_async(&conn).await?.runtime_state; - if instance_state.propolis_id.is_some() - || instance_state.nexus_state != stopped - { - return Err(TxnError::CustomError( - NetworkInterfaceUpdateError::InstanceNotStopped, - )); - } - Ok(update_target_query.get_result_async(&conn).await?) - }) + self.transaction_retry_wrapper("instance_update_network_interface") + .transaction(&conn, |conn| { + let err = err.clone(); + let stopped = stopped.clone(); + let update_target_query = update_target_query.clone(); + async move { + let instance_state = + instance_query.get_result_async(&conn).await?.runtime_state; + if instance_state.propolis_id.is_some() + || instance_state.nexus_state != stopped + { + return Err(err.bail(NetworkInterfaceUpdateError::InstanceNotStopped)); + } + update_target_query.get_result_async(&conn).await + } + }).await } - .await // Convert to `InstanceNetworkInterface` before returning, we know // this is valid as we've filtered appropriately above. .map(NetworkInterface::as_instance) - .map_err(|e| match e { - TxnError::CustomError( - NetworkInterfaceUpdateError::InstanceNotStopped, - ) => Error::invalid_request( - "Instance must be stopped to update its network interfaces", - ), - _ => Error::internal_error(&format!("Transaction error: {:?}", e)), + .map_err(|e| { + if let Some(err) = err.take() { + match err { + NetworkInterfaceUpdateError::InstanceNotStopped => { + return Error::invalid_request( + "Instance must be stopped to update its network interfaces", + ); + }, + NetworkInterfaceUpdateError::FailedToUnsetPrimary(err) => { + return public_error_from_diesel(err, ErrorHandler::Server); + }, + } + } + public_error_from_diesel(e, ErrorHandler::Server) }) } } diff --git a/nexus/db-queries/src/db/datastore/project.rs b/nexus/db-queries/src/db/datastore/project.rs index c447b5bf98..ba0c64abfd 100644 --- a/nexus/db-queries/src/db/datastore/project.rs +++ b/nexus/db-queries/src/db/datastore/project.rs @@ -13,7 +13,6 @@ use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::fixed_data::project::SERVICES_PROJECT; use crate::db::fixed_data::silo::INTERNAL_SILO_ID; use crate::db::identity::Resource; @@ -24,7 +23,8 @@ use crate::db::model::ProjectUpdate; use crate::db::model::Silo; use crate::db::model::VirtualProvisioningCollection; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use omicron_common::api::external::http_pagination::PaginatedBy; @@ -151,49 +151,62 @@ impl DataStore { use db::schema::project::dsl; + let err = OptionalError::new(); let name = project.name().as_str().to_string(); + let conn = self.pool_connection_authorized(opctx).await?; + let db_project = self - .pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - let project: Project = Silo::insert_resource( - silo_id, - diesel::insert_into(dsl::project).values(project), - ) - .insert_and_get_result_async(&conn) - .await - .map_err(|e| match e { - AsyncInsertError::CollectionNotFound => { - authz_silo_inner.not_found() - } - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Project, - &name, + .transaction_retry_wrapper("project_create_in_silo") + .transaction(&conn, |conn| { + let err = err.clone(); + + let authz_silo_inner = authz_silo_inner.clone(); + let name = name.clone(); + let project = project.clone(); + async move { + let project: Project = Silo::insert_resource( + silo_id, + diesel::insert_into(dsl::project).values(project), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => { + err.bail(authz_silo_inner.not_found()) + } + AsyncInsertError::DatabaseError(diesel_error) => err + .bail_retryable_or_else( + diesel_error, + |diesel_error| { + public_error_from_diesel( + diesel_error, + ErrorHandler::Conflict( + ResourceType::Project, + &name, + ), + ) + }, ), - ) - } - })?; - - // Create resource provisioning for the project. - self.virtual_provisioning_collection_create_on_connection( - &conn, - VirtualProvisioningCollection::new( - project.id(), - CollectionTypeProvisioned::Project, - ), - ) - .await?; - Ok(project) + })?; + + // Create resource provisioning for the project. + self.virtual_provisioning_collection_create_on_connection( + &conn, + VirtualProvisioningCollection::new( + project.id(), + CollectionTypeProvisioned::Project, + ), + ) + .await?; + Ok(project) + } }) .await - .map_err(|e| match e { - TransactionError::CustomError(e) => e, - TransactionError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + .map_err(|e| { + if let Some(err) = err.take() { + return err; } + public_error_from_diesel(e, ErrorHandler::Server) })?; Ok(( @@ -230,47 +243,56 @@ impl DataStore { use db::schema::project::dsl; - type TxnError = TransactionError; - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - let now = Utc::now(); - let updated_rows = diesel::update(dsl::project) - .filter(dsl::time_deleted.is_null()) - .filter(dsl::id.eq(authz_project.id())) - .filter(dsl::rcgen.eq(db_project.rcgen)) - .set(dsl::time_deleted.eq(now)) - .returning(Project::as_returning()) - .execute_async(&conn) - .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_project), - ) - })?; + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("project_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let now = Utc::now(); + let updated_rows = diesel::update(dsl::project) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_project.id())) + .filter(dsl::rcgen.eq(db_project.rcgen)) + .set(dsl::time_deleted.eq(now)) + .returning(Project::as_returning()) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_project, + ), + ) + }) + })?; + + if updated_rows == 0 { + return Err(err.bail(Error::InvalidRequest { + message: + "deletion failed due to concurrent modification" + .to_string(), + })); + } - if updated_rows == 0 { - return Err(TxnError::CustomError(Error::InvalidRequest { - message: - "deletion failed due to concurrent modification" - .to_string(), - })); + self.virtual_provisioning_collection_delete_on_connection( + &opctx.log, + &conn, + db_project.id(), + ) + .await?; + Ok(()) } - - self.virtual_provisioning_collection_delete_on_connection( - &conn, - db_project.id(), - ) - .await?; - Ok(()) }) .await - .map_err(|e| match e { - TxnError::CustomError(e) => e, - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + .map_err(|e| { + if let Some(err) = err.take() { + return err; } + public_error_from_diesel(e, ErrorHandler::Server) })?; Ok(()) } diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index e11377f11a..a69386cfd0 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -13,8 +13,9 @@ use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; +use crate::db::error::MaybeRetryable::*; use crate::db::fixed_data::silo::INTERNAL_SILO_ID; use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET; use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; @@ -26,6 +27,7 @@ use crate::db::model::Rack; use crate::db::model::Zpool; use crate::db::pagination::paginated; use crate::db::pool::DbConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; @@ -58,6 +60,7 @@ use omicron_common::api::external::ResourceType; use omicron_common::api::external::UpdateResult; use omicron_common::bail_unless; use std::net::IpAddr; +use std::sync::{Arc, OnceLock}; use uuid::Uuid; /// Groups arguments related to rack initialization @@ -88,18 +91,30 @@ enum RackInitError { DnsSerialization(Error), Silo(Error), RoleAssignment(Error), + // Retryable database error + Retryable(DieselError), + // Other non-retryable database error + Database(DieselError), } -type TxnError = TransactionError; -impl From for Error { - fn from(e: TxnError) -> Self { +// Catch-all for Diesel error conversion into RackInitError, which +// can also label errors as retryable. +impl From for RackInitError { + fn from(e: DieselError) -> Self { + if retryable(&e) { + Self::Retryable(e) + } else { + Self::Database(e) + } + } +} + +impl From for Error { + fn from(e: RackInitError) -> Self { match e { - TxnError::CustomError(RackInitError::AddingIp(err)) => err, - TxnError::CustomError(RackInitError::AddingNic(err)) => err, - TxnError::CustomError(RackInitError::DatasetInsert { - err, - zpool_id, - }) => match err { + RackInitError::AddingIp(err) => err, + RackInitError::AddingNic(err) => err, + RackInitError::DatasetInsert { err, zpool_id } => match err { AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { type_name: ResourceType::Zpool, lookup_type: LookupType::ById(zpool_id), @@ -108,43 +123,36 @@ impl From for Error { public_error_from_diesel(e, ErrorHandler::Server) } }, - TxnError::CustomError(RackInitError::ServiceInsert(err)) => { - Error::internal_error(&format!( - "failed to insert Service record: {:#}", - err - )) - } - TxnError::CustomError(RackInitError::RackUpdate { - err, - rack_id, - }) => public_error_from_diesel( - err, - ErrorHandler::NotFoundByLookup( - ResourceType::Rack, - LookupType::ById(rack_id), - ), + RackInitError::ServiceInsert(err) => Error::internal_error( + &format!("failed to insert Service record: {:#}", err), ), - TxnError::CustomError(RackInitError::DnsSerialization(err)) => { - Error::internal_error(&format!( - "failed to serialize initial DNS records: {:#}", - err - )) - } - TxnError::CustomError(RackInitError::Silo(err)) => { - Error::internal_error(&format!( - "failed to create recovery Silo: {:#}", - err - )) - } - TxnError::CustomError(RackInitError::RoleAssignment(err)) => { - Error::internal_error(&format!( - "failed to assign role to initial user: {:#}", - err - )) - } - TxnError::Database(e) => { - Error::internal_error(&format!("Transaction error: {}", e)) + RackInitError::RackUpdate { err, rack_id } => { + public_error_from_diesel( + err, + ErrorHandler::NotFoundByLookup( + ResourceType::Rack, + LookupType::ById(rack_id), + ), + ) } + RackInitError::DnsSerialization(err) => Error::internal_error( + &format!("failed to serialize initial DNS records: {:#}", err), + ), + RackInitError::Silo(err) => Error::internal_error(&format!( + "failed to create recovery Silo: {:#}", + err + )), + RackInitError::RoleAssignment(err) => Error::internal_error( + &format!("failed to assign role to initial user: {:#}", err), + ), + RackInitError::Retryable(err) => Error::internal_error(&format!( + "failed operation due to database contention: {:#}", + err + )), + RackInitError::Database(err) => Error::internal_error(&format!( + "failed operation due to database error: {:#}", + err + )), } } } @@ -336,9 +344,6 @@ impl DataStore { Ok(()) } - // The following methods which return a `TxnError` take a `conn` parameter - // which comes from the transaction created in `rack_set_initialized`. - #[allow(clippy::too_many_arguments)] async fn rack_create_recovery_silo( &self, @@ -350,7 +355,7 @@ impl DataStore { recovery_user_id: external_params::UserId, recovery_user_password_hash: omicron_passwords::PasswordHashString, dns_update: DnsVersionUpdateBuilder, - ) -> Result<(), TxnError> { + ) -> Result<(), RackInitError> { let db_silo = self .silo_create_conn( conn, @@ -361,8 +366,10 @@ impl DataStore { dns_update, ) .await - .map_err(RackInitError::Silo) - .map_err(TxnError::CustomError)?; + .map_err(|err| match err.retryable() { + NotRetryable(err) => RackInitError::Silo(err.into()), + Retryable(err) => RackInitError::Retryable(err), + })?; info!(log, "Created recovery silo"); // Create the first user in the initial Recovery Silo @@ -416,8 +423,7 @@ impl DataStore { }], ) .await - .map_err(RackInitError::RoleAssignment) - .map_err(TxnError::CustomError)?; + .map_err(RackInitError::RoleAssignment)?; debug!(log, "Generated role assignment queries"); q1.execute_async(conn).await?; @@ -433,7 +439,7 @@ impl DataStore { log: &slog::Logger, service_pool: &db::model::IpPool, service: internal_params::ServicePutRequest, - ) -> Result<(), TxnError> { + ) -> Result<(), RackInitError> { use internal_params::ServiceKind; let service_db = db::model::Service::new( @@ -443,9 +449,12 @@ impl DataStore { service.address, service.kind.clone().into(), ); - self.service_upsert_conn(conn, service_db).await.map_err(|e| { - TxnError::CustomError(RackInitError::ServiceInsert(e)) - })?; + self.service_upsert_conn(conn, service_db).await.map_err( + |e| match e.retryable() { + Retryable(e) => RackInitError::Retryable(e), + NotRetryable(e) => RackInitError::ServiceInsert(e.into()), + }, + )?; // For services with external connectivity, we record their // explicit IP allocation and create a service NIC as well. @@ -476,9 +485,7 @@ impl DataStore { Some(nic.ip), Some(nic.mac), ) - .map_err(|e| { - TxnError::CustomError(RackInitError::AddingNic(e)) - })?; + .map_err(|e| RackInitError::AddingNic(e))?; Some((db_ip, db_nic)) } ServiceKind::BoundaryNtp { snat, ref nic } => { @@ -500,9 +507,7 @@ impl DataStore { Some(nic.ip), Some(nic.mac), ) - .map_err(|e| { - TxnError::CustomError(RackInitError::AddingNic(e)) - })?; + .map_err(|e| RackInitError::AddingNic(e))?; Some((db_ip, db_nic)) } _ => None, @@ -517,7 +522,10 @@ impl DataStore { IP address for {}", service.kind, ); - TxnError::CustomError(RackInitError::AddingIp(err)) + match err.retryable() { + Retryable(e) => RackInitError::Retryable(e), + NotRetryable(e) => RackInitError::AddingIp(e.into()), + } })?; self.create_network_interface_raw_conn(conn, db_nic) @@ -530,9 +538,10 @@ impl DataStore { _, db::model::NetworkInterfaceKind::Service, ) => Ok(()), - _ => Err(TxnError::CustomError( - RackInitError::AddingNic(e.into_external()), - )), + InsertError::Retryable(err) => { + Err(RackInitError::Retryable(err)) + } + _ => Err(RackInitError::AddingNic(e.into_external())), } })?; } @@ -551,146 +560,187 @@ impl DataStore { opctx.authorize(authz::Action::CreateChild, &authz::FLEET).await?; - let rack_id = rack_init.rack_id; - let services = rack_init.services; - let datasets = rack_init.datasets; - let service_ip_pool_ranges = rack_init.service_ip_pool_ranges; - let internal_dns = rack_init.internal_dns; - let external_dns = rack_init.external_dns; - let (authz_service_pool, service_pool) = self.ip_pools_service_lookup(&opctx).await?; // NOTE: This operation could likely be optimized with a CTE, but given // the low-frequency of calls, this optimization has been deferred. let log = opctx.log.clone(); + let err = Arc::new(OnceLock::new()); + + // NOTE: This transaction cannot yet be made retryable, as it uses + // nested transactions. let rack = self .pool_connection_authorized(opctx) .await? - .transaction_async(|conn| async move { - // Early exit if the rack has already been initialized. - let rack = rack_dsl::rack - .filter(rack_dsl::id.eq(rack_id)) - .select(Rack::as_select()) - .get_result_async(&conn) - .await - .map_err(|e| { - warn!(log, "Initializing Rack: Rack UUID not found"); - TxnError::CustomError(RackInitError::RackUpdate { - err: e, - rack_id, - }) - })?; - if rack.initialized { - info!(log, "Early exit: Rack already initialized"); - return Ok(rack); - } + .transaction_async(|conn| { + let err = err.clone(); + let log = log.clone(); + let authz_service_pool = authz_service_pool.clone(); + let rack_init = rack_init.clone(); + let service_pool = service_pool.clone(); + async move { + let rack_id = rack_init.rack_id; + let services = rack_init.services; + let datasets = rack_init.datasets; + let service_ip_pool_ranges = rack_init.service_ip_pool_ranges; + let internal_dns = rack_init.internal_dns; + let external_dns = rack_init.external_dns; + + // Early exit if the rack has already been initialized. + let rack = rack_dsl::rack + .filter(rack_dsl::id.eq(rack_id)) + .select(Rack::as_select()) + .get_result_async(&conn) + .await + .map_err(|e| { + warn!(log, "Initializing Rack: Rack UUID not found"); + err.set(RackInitError::RackUpdate { + err: e, + rack_id, + }).unwrap(); + DieselError::RollbackTransaction + })?; + if rack.initialized { + info!(log, "Early exit: Rack already initialized"); + return Ok::<_, DieselError>(rack); + } - // Otherwise, insert services and datasets. + // Otherwise, insert services and datasets. - // Set up the IP pool for internal services. - for range in service_ip_pool_ranges { - Self::ip_pool_add_range_on_connection( - &conn, - opctx, - &authz_service_pool, - &range, - ) - .await - .map_err(|err| { - warn!( - log, - "Initializing Rack: Failed to add IP pool range" - ); - TxnError::CustomError(RackInitError::AddingIp(err)) - })?; - } + // Set up the IP pool for internal services. + for range in service_ip_pool_ranges { + Self::ip_pool_add_range_on_connection( + &conn, + opctx, + &authz_service_pool, + &range, + ) + .await + .map_err(|e| { + warn!( + log, + "Initializing Rack: Failed to add IP pool range" + ); + err.set(RackInitError::AddingIp(e)).unwrap(); + DieselError::RollbackTransaction + })?; + } + + // Allocate records for all services. + for service in services { + self.rack_populate_service_records( + &conn, + &log, + &service_pool, + service, + ) + .await + .map_err(|e| { + err.set(e).unwrap(); + DieselError::RollbackTransaction + })?; + } + info!(log, "Inserted services"); + + for dataset in datasets { + use db::schema::dataset::dsl; + let zpool_id = dataset.pool_id; + >::insert_resource( + zpool_id, + diesel::insert_into(dsl::dataset) + .values(dataset.clone()) + .on_conflict(dsl::id) + .do_update() + .set(( + dsl::time_modified.eq(Utc::now()), + dsl::pool_id.eq(excluded(dsl::pool_id)), + dsl::ip.eq(excluded(dsl::ip)), + dsl::port.eq(excluded(dsl::port)), + dsl::kind.eq(excluded(dsl::kind)), + )), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| { + err.set(RackInitError::DatasetInsert { + err: e, + zpool_id, + }).unwrap(); + DieselError::RollbackTransaction + })?; + } + info!(log, "Inserted datasets"); - // Allocate records for all services. - for service in services { - self.rack_populate_service_records( + // Insert the initial contents of the internal and external DNS + // zones. + Self::load_dns_data(&conn, internal_dns) + .await + .map_err(|e| { + err.set(RackInitError::DnsSerialization(e)).unwrap(); + DieselError::RollbackTransaction + })?; + info!(log, "Populated DNS tables for internal DNS"); + + Self::load_dns_data(&conn, external_dns) + .await + .map_err(|e| { + err.set(RackInitError::DnsSerialization(e)).unwrap(); + DieselError::RollbackTransaction + })?; + info!(log, "Populated DNS tables for external DNS"); + + // Create the initial Recovery Silo + self.rack_create_recovery_silo( + &opctx, &conn, &log, - &service_pool, - service, - ) - .await?; - } - info!(log, "Inserted services"); - - for dataset in datasets { - use db::schema::dataset::dsl; - let zpool_id = dataset.pool_id; - >::insert_resource( - zpool_id, - diesel::insert_into(dsl::dataset) - .values(dataset.clone()) - .on_conflict(dsl::id) - .do_update() - .set(( - dsl::time_modified.eq(Utc::now()), - dsl::pool_id.eq(excluded(dsl::pool_id)), - dsl::ip.eq(excluded(dsl::ip)), - dsl::port.eq(excluded(dsl::port)), - dsl::kind.eq(excluded(dsl::kind)), - )), + rack_init.recovery_silo, + rack_init.recovery_silo_fq_dns_name, + rack_init.recovery_user_id, + rack_init.recovery_user_password_hash, + rack_init.dns_update, ) - .insert_and_get_result_async(&conn) .await - .map_err(|err| { - TxnError::CustomError(RackInitError::DatasetInsert { - err, - zpool_id, - }) + .map_err(|e| match e { + RackInitError::Retryable(e) => e, + _ => { + err.set(e).unwrap(); + DieselError::RollbackTransaction + }, })?; - } - info!(log, "Inserted datasets"); - - // Insert the initial contents of the internal and external DNS - // zones. - Self::load_dns_data(&conn, internal_dns) - .await - .map_err(RackInitError::DnsSerialization) - .map_err(TxnError::CustomError)?; - info!(log, "Populated DNS tables for internal DNS"); - Self::load_dns_data(&conn, external_dns) - .await - .map_err(RackInitError::DnsSerialization) - .map_err(TxnError::CustomError)?; - info!(log, "Populated DNS tables for external DNS"); - - // Create the initial Recovery Silo - self.rack_create_recovery_silo( - &opctx, - &conn, - &log, - rack_init.recovery_silo, - rack_init.recovery_silo_fq_dns_name, - rack_init.recovery_user_id, - rack_init.recovery_user_password_hash, - rack_init.dns_update, - ) - .await?; - - let rack = diesel::update(rack_dsl::rack) - .filter(rack_dsl::id.eq(rack_id)) - .set(( - rack_dsl::initialized.eq(true), - rack_dsl::time_modified.eq(Utc::now()), - )) - .returning(Rack::as_returning()) - .get_result_async::(&conn) - .await - .map_err(|err| { - TxnError::CustomError(RackInitError::RackUpdate { - err, - rack_id, - }) - })?; - Ok::<_, TxnError>(rack) - }) - .await?; + let rack = diesel::update(rack_dsl::rack) + .filter(rack_dsl::id.eq(rack_id)) + .set(( + rack_dsl::initialized.eq(true), + rack_dsl::time_modified.eq(Utc::now()), + )) + .returning(Rack::as_returning()) + .get_result_async::(&conn) + .await + .map_err(|e| { + if retryable(&e) { + return e; + } + err.set(RackInitError::RackUpdate { + err: e, + rack_id, + }).unwrap(); + DieselError::RollbackTransaction + })?; + Ok(rack) + } + }, + ) + .await + .map_err(|e| { + if let Some(err) = Arc::try_unwrap(err).unwrap().take() { + err.into() + } else { + Error::internal_error(&format!("Transaction error: {}", e)) + } + })?; Ok(rack) } @@ -745,42 +795,54 @@ impl DataStore { use crate::db::schema::external_ip::dsl as extip_dsl; use crate::db::schema::service::dsl as service_dsl; - type TxnError = TransactionError; - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - let ips = extip_dsl::external_ip - .inner_join( - service_dsl::service.on(service_dsl::id - .eq(extip_dsl::parent_id.assume_not_null())), - ) - .filter(extip_dsl::parent_id.is_not_null()) - .filter(extip_dsl::time_deleted.is_null()) - .filter(extip_dsl::is_service) - .filter(service_dsl::kind.eq(db::model::ServiceKind::Nexus)) - .select(ExternalIp::as_select()) - .get_results_async(&conn) - .await? - .into_iter() - .map(|external_ip| external_ip.ip.ip()) - .collect(); - - let dns_zones = self - .dns_zones_list_all_on_connection( - opctx, - &conn, - DnsGroup::External, - ) - .await?; - Ok((ips, dns_zones)) + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("nexus_external_addresses") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let ips = extip_dsl::external_ip + .inner_join( + service_dsl::service.on(service_dsl::id + .eq(extip_dsl::parent_id.assume_not_null())), + ) + .filter(extip_dsl::parent_id.is_not_null()) + .filter(extip_dsl::time_deleted.is_null()) + .filter(extip_dsl::is_service) + .filter( + service_dsl::kind.eq(db::model::ServiceKind::Nexus), + ) + .select(ExternalIp::as_select()) + .get_results_async(&conn) + .await? + .into_iter() + .map(|external_ip| external_ip.ip.ip()) + .collect(); + + let dns_zones = self + .dns_zones_list_all_on_connection( + opctx, + &conn, + DnsGroup::External, + ) + .await + .map_err(|e| match e.retryable() { + NotRetryable(not_retryable_err) => { + err.bail(not_retryable_err) + } + Retryable(retryable_err) => retryable_err, + })?; + + Ok((ips, dns_zones)) + } }) .await - .map_err(|error: TxnError| match error { - TransactionError::CustomError(err) => err, - TransactionError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + .map_err(|e| { + if let Some(err) = err.take() { + return err.into(); } + public_error_from_diesel(e, ErrorHandler::Server) }) } } @@ -1014,14 +1076,16 @@ mod test { async fn [](db: &DataStore) -> Vec<$model> { use crate::db::schema::$table::dsl; use nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL; - db.pool_connection_for_tests() + let conn = db.pool_connection_for_tests() .await - .unwrap() - .transaction_async(|conn| async move { + .unwrap(); + + db.transaction_retry_wrapper(concat!("fn_to_get_all_", stringify!($table))) + .transaction(&conn, |conn| async move { conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL) .await .unwrap(); - Ok::<_, crate::db::TransactionError<()>>( + Ok( dsl::$table .select($model::as_select()) .get_results_async(&conn) diff --git a/nexus/db-queries/src/db/datastore/region.rs b/nexus/db-queries/src/db/datastore/region.rs index 9465fe2792..b055a3e85c 100644 --- a/nexus/db-queries/src/db/datastore/region.rs +++ b/nexus/db-queries/src/db/datastore/region.rs @@ -10,18 +10,16 @@ use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::lookup::LookupPath; use crate::db::model::Dataset; use crate::db::model::Region; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use nexus_types::external_api::params; use omicron_common::api::external; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; -use omicron_common::backoff::{self, BackoffError}; use omicron_common::nexus_config::RegionAllocationStrategy; use slog::Logger; use uuid::Uuid; @@ -152,7 +150,7 @@ impl DataStore { /// Also updates the storage usage on their corresponding datasets. pub async fn regions_hard_delete( &self, - log: &Logger, + _log: &Logger, region_ids: Vec, ) -> DeleteResult { if region_ids.is_empty() { @@ -164,98 +162,79 @@ impl DataStore { #[error("Numeric error: {0}")] NumericError(String), } - type TxnError = TransactionError; - - // Retry this transaction until it succeeds. It's a little heavy in that - // there's a for loop inside that iterates over the datasets the - // argument regions belong to, and it often encounters the "retry - // transaction" error. - let transaction = { - |region_ids: Vec| async { - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - use db::schema::dataset::dsl as dataset_dsl; - use db::schema::region::dsl as region_dsl; - - // Remove the regions, collecting datasets they're from. - let datasets = diesel::delete(region_dsl::region) - .filter(region_dsl::id.eq_any(region_ids)) - .returning(region_dsl::dataset_id) - .get_results_async::(&conn).await?; - - // Update datasets to which the regions belonged. - for dataset in datasets { - let dataset_total_occupied_size: Option< - diesel::pg::data_types::PgNumeric, - > = region_dsl::region - .filter(region_dsl::dataset_id.eq(dataset)) - .select(diesel::dsl::sum( - region_dsl::block_size - * region_dsl::blocks_per_extent - * region_dsl::extent_count, - )) - .nullable() - .get_result_async(&conn).await?; - - let dataset_total_occupied_size: i64 = if let Some( - dataset_total_occupied_size, - ) = - dataset_total_occupied_size - { - let dataset_total_occupied_size: db::model::ByteCount = - dataset_total_occupied_size.try_into().map_err( - |e: anyhow::Error| { - TxnError::CustomError( - RegionDeleteError::NumericError( - e.to_string(), - ), - ) - }, - )?; - - dataset_total_occupied_size.into() - } else { - 0 - }; - - diesel::update(dataset_dsl::dataset) - .filter(dataset_dsl::id.eq(dataset)) - .set( - dataset_dsl::size_used - .eq(dataset_total_occupied_size), - ) - .execute_async(&conn).await?; - } - - Ok(()) - }) - .await - .map_err(|e: TxnError| { - if e.retry_transaction() { - BackoffError::transient(Error::internal_error( - &format!("Retryable transaction error {:?}", e) + let err = OptionalError::new(); + let conn = self.pool_connection_unauthorized().await?; + self.transaction_retry_wrapper("regions_hard_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + let region_ids = region_ids.clone(); + async move { + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region::dsl as region_dsl; + + // Remove the regions, collecting datasets they're from. + let datasets = diesel::delete(region_dsl::region) + .filter(region_dsl::id.eq_any(region_ids)) + .returning(region_dsl::dataset_id) + .get_results_async::(&conn).await?; + + // Update datasets to which the regions belonged. + for dataset in datasets { + let dataset_total_occupied_size: Option< + diesel::pg::data_types::PgNumeric, + > = region_dsl::region + .filter(region_dsl::dataset_id.eq(dataset)) + .select(diesel::dsl::sum( + region_dsl::block_size + * region_dsl::blocks_per_extent + * region_dsl::extent_count, )) + .nullable() + .get_result_async(&conn).await?; + + let dataset_total_occupied_size: i64 = if let Some( + dataset_total_occupied_size, + ) = + dataset_total_occupied_size + { + let dataset_total_occupied_size: db::model::ByteCount = + dataset_total_occupied_size.try_into().map_err( + |e: anyhow::Error| { + err.bail(RegionDeleteError::NumericError( + e.to_string(), + )) + }, + )?; + + dataset_total_occupied_size.into() } else { - BackoffError::Permanent(Error::internal_error( - &format!("Transaction error: {}", e) - )) + 0 + }; + + diesel::update(dataset_dsl::dataset) + .filter(dataset_dsl::id.eq(dataset)) + .set( + dataset_dsl::size_used + .eq(dataset_total_occupied_size), + ) + .execute_async(&conn).await?; + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + RegionDeleteError::NumericError(err) => { + return Error::internal_error( + &format!("Transaction error: {}", err) + ); } - }) - } - }; - - backoff::retry_notify( - backoff::retry_policy_internal_service_aggressive(), - || async { - let region_ids = region_ids.clone(); - transaction(region_ids).await - }, - |e: Error, delay| { - info!(log, "{:?}, trying again in {:?}", e, delay,); - }, - ) - .await + } + } + public_error_from_diesel(e, ErrorHandler::Server) + }) } /// Return the total occupied size for a dataset diff --git a/nexus/db-queries/src/db/datastore/service.rs b/nexus/db-queries/src/db/datastore/service.rs index 40bf250abe..df7ed27a6d 100644 --- a/nexus/db-queries/src/db/datastore/service.rs +++ b/nexus/db-queries/src/db/datastore/service.rs @@ -11,7 +11,9 @@ use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; +use crate::db::error::TransactionError; use crate::db::identity::Asset; use crate::db::model::Service; use crate::db::model::Sled; @@ -38,7 +40,12 @@ impl DataStore { service: Service, ) -> CreateResult { let conn = self.pool_connection_authorized(opctx).await?; - self.service_upsert_conn(&conn, service).await + self.service_upsert_conn(&conn, service).await.map_err(|e| match e { + TransactionError::CustomError(err) => err, + TransactionError::Database(err) => { + public_error_from_diesel(err, ErrorHandler::Server) + } + }) } /// Stores a new service in the database (using an existing db connection). @@ -46,7 +53,7 @@ impl DataStore { &self, conn: &async_bb8_diesel::Connection, service: Service, - ) -> CreateResult { + ) -> Result> { use db::schema::service::dsl; let service_id = service.id(); @@ -68,17 +75,24 @@ impl DataStore { .insert_and_get_result_async(conn) .await .map_err(|e| match e { - AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { - type_name: ResourceType::Sled, - lookup_type: LookupType::ById(sled_id), - }, - AsyncInsertError::DatabaseError(e) => public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Service, - &service_id.to_string(), - ), - ), + AsyncInsertError::CollectionNotFound => { + TransactionError::CustomError(Error::ObjectNotFound { + type_name: ResourceType::Sled, + lookup_type: LookupType::ById(sled_id), + }) + } + AsyncInsertError::DatabaseError(e) => { + if retryable(&e) { + return TransactionError::Database(e); + } + TransactionError::CustomError(public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Service, + &service_id.to_string(), + ), + )) + } }) } diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs index ec3658c067..ab48ec458f 100644 --- a/nexus/db-queries/src/db/datastore/silo.rs +++ b/nexus/db-queries/src/db/datastore/silo.rs @@ -11,6 +11,7 @@ use crate::context::OpContext; use crate::db; use crate::db::datastore::RunnableQuery; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::fixed_data::silo::{DEFAULT_SILO, INTERNAL_SILO}; @@ -123,15 +124,17 @@ impl DataStore { dns_update: DnsVersionUpdateBuilder, ) -> CreateResult { let conn = self.pool_connection_authorized(opctx).await?; - self.silo_create_conn( - &conn, - opctx, - nexus_opctx, - new_silo_params, - new_silo_dns_names, - dns_update, - ) - .await + let silo = self + .silo_create_conn( + &conn, + opctx, + nexus_opctx, + new_silo_params, + new_silo_dns_names, + dns_update, + ) + .await?; + Ok(silo) } pub async fn silo_create_conn( @@ -142,7 +145,7 @@ impl DataStore { new_silo_params: params::SiloCreate, new_silo_dns_names: &[String], dns_update: DnsVersionUpdateBuilder, - ) -> CreateResult { + ) -> Result> { let silo_id = Uuid::new_v4(); let silo_group_id = Uuid::new_v4(); @@ -199,71 +202,71 @@ impl DataStore { None }; - conn.transaction_async(|conn| async move { - let silo = silo_create_query - .get_result_async(&conn) - .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Silo, - new_silo_params.identity.name.as_str(), - ), - ) - })?; - self.virtual_provisioning_collection_create_on_connection( - &conn, - VirtualProvisioningCollection::new( - silo.id(), - CollectionTypeProvisioned::Silo, - ), - ) - .await?; - - if let Some(query) = silo_admin_group_ensure_query { - query.get_result_async(&conn).await?; - } - - if let Some(queries) = silo_admin_group_role_assignment_queries { - let (delete_old_query, insert_new_query) = queries; - delete_old_query.execute_async(&conn).await?; - insert_new_query.execute_async(&conn).await?; - } - - let certificates = new_silo_params - .tls_certificates - .into_iter() - .map(|c| { - Certificate::new( + let silo = conn + .transaction_async(|conn| async move { + let silo = silo_create_query + .get_result_async(&conn) + .await + .map_err(|e| { + if retryable(&e) { + return TransactionError::Database(e); + } + TransactionError::CustomError(public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Silo, + new_silo_params.identity.name.as_str(), + ), + )) + })?; + self.virtual_provisioning_collection_create_on_connection( + &conn, + VirtualProvisioningCollection::new( silo.id(), - Uuid::new_v4(), - ServiceKind::Nexus, - c, - new_silo_dns_names, - ) - }) - .collect::, _>>() - .map_err(Error::from)?; - { - use db::schema::certificate::dsl; - diesel::insert_into(dsl::certificate) - .values(certificates) - .execute_async(&conn) - .await?; - } - - self.dns_update(nexus_opctx, &conn, dns_update).await?; + CollectionTypeProvisioned::Silo, + ), + ) + .await?; - Ok(silo) - }) - .await - .map_err(|e| match e { - TransactionError::CustomError(e) => e, - TransactionError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + if let Some(query) = silo_admin_group_ensure_query { + query.get_result_async(&conn).await?; + } + + if let Some(queries) = silo_admin_group_role_assignment_queries + { + let (delete_old_query, insert_new_query) = queries; + delete_old_query.execute_async(&conn).await?; + insert_new_query.execute_async(&conn).await?; + } + + let certificates = new_silo_params + .tls_certificates + .into_iter() + .map(|c| { + Certificate::new( + silo.id(), + Uuid::new_v4(), + ServiceKind::Nexus, + c, + new_silo_dns_names, + ) + }) + .collect::, _>>() + .map_err(Error::from)?; + { + use db::schema::certificate::dsl; + diesel::insert_into(dsl::certificate) + .values(certificates) + .execute_async(&conn) + .await?; + } + + self.dns_update(nexus_opctx, &conn, dns_update).await?; + + Ok::>(silo) + }) + .await?; + Ok(silo) } pub async fn silos_list_by_id( @@ -380,7 +383,7 @@ impl DataStore { } self.virtual_provisioning_collection_delete_on_connection( - &conn, id, + &opctx.log, &conn, id, ) .await?; diff --git a/nexus/db-queries/src/db/datastore/silo_group.rs b/nexus/db-queries/src/db/datastore/silo_group.rs index 46f4aae7c9..29fcb7490b 100644 --- a/nexus/db-queries/src/db/datastore/silo_group.rs +++ b/nexus/db-queries/src/db/datastore/silo_group.rs @@ -145,35 +145,39 @@ impl DataStore { ) -> UpdateResult<()> { opctx.authorize(authz::Action::Modify, authz_silo_user).await?; - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - use db::schema::silo_group_membership::dsl; + let conn = self.pool_connection_authorized(opctx).await?; - // Delete existing memberships for user - let silo_user_id = authz_silo_user.id(); - diesel::delete(dsl::silo_group_membership) - .filter(dsl::silo_user_id.eq(silo_user_id)) - .execute_async(&conn) - .await?; + self.transaction_retry_wrapper("silo_group_membership_replace_for_user") + .transaction(&conn, |conn| { + let silo_group_ids = silo_group_ids.clone(); + async move { + use db::schema::silo_group_membership::dsl; + + // Delete existing memberships for user + let silo_user_id = authz_silo_user.id(); + diesel::delete(dsl::silo_group_membership) + .filter(dsl::silo_user_id.eq(silo_user_id)) + .execute_async(&conn) + .await?; - // Create new memberships for user - let silo_group_memberships: Vec< - db::model::SiloGroupMembership, - > = silo_group_ids - .iter() - .map(|group_id| db::model::SiloGroupMembership { - silo_group_id: *group_id, - silo_user_id, - }) - .collect(); + // Create new memberships for user + let silo_group_memberships: Vec< + db::model::SiloGroupMembership, + > = silo_group_ids + .iter() + .map(|group_id| db::model::SiloGroupMembership { + silo_group_id: *group_id, + silo_user_id, + }) + .collect(); - diesel::insert_into(dsl::silo_group_membership) - .values(silo_group_memberships) - .execute_async(&conn) - .await?; + diesel::insert_into(dsl::silo_group_membership) + .values(silo_group_memberships) + .execute_async(&conn) + .await?; - Ok(()) + Ok(()) + } }) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 406119a636..023384a9bf 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -10,13 +10,12 @@ use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::Sled; use crate::db::model::SledResource; use crate::db::model::SledUpdate; use crate::db::pagination::paginated; use crate::db::update_and_check::UpdateAndCheck; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; @@ -90,123 +89,141 @@ impl DataStore { enum SledReservationError { NotFound, } - type TxnError = TransactionError; - - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - use db::schema::sled_resource::dsl as resource_dsl; - // Check if resource ID already exists - if so, return it. - let old_resource = resource_dsl::sled_resource - .filter(resource_dsl::id.eq(resource_id)) - .select(SledResource::as_select()) - .limit(1) - .load_async(&conn) - .await?; - - if !old_resource.is_empty() { - return Ok(old_resource[0].clone()); - } - - // If it doesn't already exist, find a sled with enough space - // for the resources we're requesting. - use db::schema::sled::dsl as sled_dsl; - // This answers the boolean question: - // "Does the SUM of all hardware thread usage, plus the one we're trying - // to allocate, consume less threads than exists on the sled?" - let sled_has_space_for_threads = - (diesel::dsl::sql::(&format!( - "COALESCE(SUM(CAST({} as INT8)), 0)", - resource_dsl::hardware_threads::NAME - )) + resources.hardware_threads) - .le(sled_dsl::usable_hardware_threads); - - // This answers the boolean question: - // "Does the SUM of all RAM usage, plus the one we're trying - // to allocate, consume less RAM than exists on the sled?" - let sled_has_space_for_rss = - (diesel::dsl::sql::(&format!( - "COALESCE(SUM(CAST({} as INT8)), 0)", - resource_dsl::rss_ram::NAME - )) + resources.rss_ram) - .le(sled_dsl::usable_physical_ram); - - // Determine whether adding this service's reservoir allocation - // to what's allocated on the sled would avoid going over quota. - let sled_has_space_in_reservoir = - (diesel::dsl::sql::(&format!( - "COALESCE(SUM(CAST({} as INT8)), 0)", - resource_dsl::reservoir_ram::NAME - )) + resources.reservoir_ram) - .le(sled_dsl::reservoir_size); - - // Generate a query describing all of the sleds that have space - // for this reservation. - let mut sled_targets = sled_dsl::sled - .left_join( - resource_dsl::sled_resource - .on(resource_dsl::sled_id.eq(sled_dsl::id)), - ) - .group_by(sled_dsl::id) - .having( - sled_has_space_for_threads - .and(sled_has_space_for_rss) - .and(sled_has_space_in_reservoir), - ) - .filter(sled_dsl::time_deleted.is_null()) - // Filter out sleds that are not provisionable. - .filter( - sled_dsl::provision_state - .eq(db::model::SledProvisionState::Provisionable), - ) - .select(sled_dsl::id) - .into_boxed(); - - // Further constrain the sled IDs according to any caller- - // supplied constraints. - if let Some(must_select_from) = constraints.must_select_from() { - sled_targets = sled_targets - .filter(sled_dsl::id.eq_any(must_select_from.to_vec())); - } - sql_function!(fn random() -> diesel::sql_types::Float); - let sled_targets = sled_targets - .order(random()) - .limit(1) - .get_results_async::(&conn) - .await?; - - if sled_targets.is_empty() { - return Err(TxnError::CustomError( - SledReservationError::NotFound, - )); + let err = OptionalError::new(); + + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("sled_reservation_create") + .transaction(&conn, |conn| { + // Clone variables into retryable function + let err = err.clone(); + let constraints = constraints.clone(); + let resources = resources.clone(); + + async move { + use db::schema::sled_resource::dsl as resource_dsl; + // Check if resource ID already exists - if so, return it. + let old_resource = resource_dsl::sled_resource + .filter(resource_dsl::id.eq(resource_id)) + .select(SledResource::as_select()) + .limit(1) + .load_async(&conn) + .await?; + + if !old_resource.is_empty() { + return Ok(old_resource[0].clone()); + } + + // If it doesn't already exist, find a sled with enough space + // for the resources we're requesting. + use db::schema::sled::dsl as sled_dsl; + // This answers the boolean question: + // "Does the SUM of all hardware thread usage, plus the one we're trying + // to allocate, consume less threads than exists on the sled?" + let sled_has_space_for_threads = + (diesel::dsl::sql::( + &format!( + "COALESCE(SUM(CAST({} as INT8)), 0)", + resource_dsl::hardware_threads::NAME + ), + ) + resources.hardware_threads) + .le(sled_dsl::usable_hardware_threads); + + // This answers the boolean question: + // "Does the SUM of all RAM usage, plus the one we're trying + // to allocate, consume less RAM than exists on the sled?" + let sled_has_space_for_rss = + (diesel::dsl::sql::( + &format!( + "COALESCE(SUM(CAST({} as INT8)), 0)", + resource_dsl::rss_ram::NAME + ), + ) + resources.rss_ram) + .le(sled_dsl::usable_physical_ram); + + // Determine whether adding this service's reservoir allocation + // to what's allocated on the sled would avoid going over quota. + let sled_has_space_in_reservoir = + (diesel::dsl::sql::( + &format!( + "COALESCE(SUM(CAST({} as INT8)), 0)", + resource_dsl::reservoir_ram::NAME + ), + ) + resources.reservoir_ram) + .le(sled_dsl::reservoir_size); + + // Generate a query describing all of the sleds that have space + // for this reservation. + let mut sled_targets = + sled_dsl::sled + .left_join( + resource_dsl::sled_resource + .on(resource_dsl::sled_id.eq(sled_dsl::id)), + ) + .group_by(sled_dsl::id) + .having( + sled_has_space_for_threads + .and(sled_has_space_for_rss) + .and(sled_has_space_in_reservoir), + ) + .filter(sled_dsl::time_deleted.is_null()) + // Filter out sleds that are not provisionable. + .filter(sled_dsl::provision_state.eq( + db::model::SledProvisionState::Provisionable, + )) + .select(sled_dsl::id) + .into_boxed(); + + // Further constrain the sled IDs according to any caller- + // supplied constraints. + if let Some(must_select_from) = + constraints.must_select_from() + { + sled_targets = sled_targets.filter( + sled_dsl::id.eq_any(must_select_from.to_vec()), + ); + } + + sql_function!(fn random() -> diesel::sql_types::Float); + let sled_targets = sled_targets + .order(random()) + .limit(1) + .get_results_async::(&conn) + .await?; + + if sled_targets.is_empty() { + return Err(err.bail(SledReservationError::NotFound)); + } + + // Create a SledResource record, associate it with the target + // sled. + let resource = SledResource::new( + resource_id, + sled_targets[0], + resource_kind, + resources, + ); + + diesel::insert_into(resource_dsl::sled_resource) + .values(resource) + .returning(SledResource::as_returning()) + .get_result_async(&conn) + .await } - - // Create a SledResource record, associate it with the target - // sled. - let resource = SledResource::new( - resource_id, - sled_targets[0], - resource_kind, - resources, - ); - - Ok(diesel::insert_into(resource_dsl::sled_resource) - .values(resource) - .returning(SledResource::as_returning()) - .get_result_async(&conn) - .await?) }) .await - .map_err(|e| match e { - TxnError::CustomError(SledReservationError::NotFound) => { - external::Error::unavail( - "No sleds can fit the requested instance", - ) - } - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SledReservationError::NotFound => { + return external::Error::unavail( + "No sleds can fit the requested instance", + ); + } + } } + public_error_from_diesel(e, ErrorHandler::Server) }) } diff --git a/nexus/db-queries/src/db/datastore/snapshot.rs b/nexus/db-queries/src/db/datastore/snapshot.rs index 7c03e4bd40..7a9eb8d2bc 100644 --- a/nexus/db-queries/src/db/datastore/snapshot.rs +++ b/nexus/db-queries/src/db/datastore/snapshot.rs @@ -20,8 +20,7 @@ use crate::db::model::SnapshotState; use crate::db::pagination::paginated; use crate::db::update_and_check::UpdateAndCheck; use crate::db::update_and_check::UpdateStatus; -use crate::db::TransactionError; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; @@ -48,114 +47,99 @@ impl DataStore { let gen = snapshot.gen; opctx.authorize(authz::Action::CreateChild, authz_project).await?; - #[derive(Debug, thiserror::Error)] - pub enum CustomError { - #[error("Resource already exists")] - ResourceAlreadyExists, - - #[error("saw AsyncInsertError")] - InsertError(AsyncInsertError), - } - - type TxnError = TransactionError; - - let snapshot_name = snapshot.name().to_string(); let project_id = snapshot.project_id; - let snapshot: Snapshot = self - .pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - use db::schema::snapshot::dsl; + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; - // If an undeleted snapshot exists in the database with the - // same name and project but a different id to the snapshot - // this function was passed as an argument, then return an - // error here. - // - // As written below, - // - // .on_conflict((dsl::project_id, dsl::name)) - // .filter_target(dsl::time_deleted.is_null()) - // .do_update() - // .set(dsl::time_modified.eq(dsl::time_modified)) - // - // will set any existing record's `time_modified` if the - // project id and name match, even if the snapshot ID does - // not match. diesel supports adding a filter below like so - // (marked with >>): - // - // .on_conflict((dsl::project_id, dsl::name)) - // .filter_target(dsl::time_deleted.is_null()) - // .do_update() - // .set(dsl::time_modified.eq(dsl::time_modified)) - // >> .filter(dsl::id.eq(snapshot.id())) - // - // which will restrict the `insert_into`'s set so that it - // only applies if the snapshot ID matches. But, - // AsyncInsertError does not have a ObjectAlreadyExists - // variant, so this will be returned as CollectionNotFound - // due to the `insert_into` failing. - // - // If this function is passed a snapshot with an ID that - // does not match, but a project and name that does, return - // ObjectAlreadyExists here. + let snapshot: Snapshot = self + .transaction_retry_wrapper("project_ensure_snapshot") + .transaction(&conn, |conn| { + let err = err.clone(); + let snapshot = snapshot.clone(); + let snapshot_name = snapshot.name().to_string(); + async move { + use db::schema::snapshot::dsl; - let existing_snapshot_id: Option = dsl::snapshot - .filter(dsl::time_deleted.is_null()) - .filter(dsl::name.eq(snapshot.name().to_string())) - .filter(dsl::project_id.eq(snapshot.project_id)) - .select(dsl::id) - .limit(1) - .first_async(&conn) - .await - .optional()?; + // If an undeleted snapshot exists in the database with the + // same name and project but a different id to the snapshot + // this function was passed as an argument, then return an + // error here. + // + // As written below, + // + // .on_conflict((dsl::project_id, dsl::name)) + // .filter_target(dsl::time_deleted.is_null()) + // .do_update() + // .set(dsl::time_modified.eq(dsl::time_modified)) + // + // will set any existing record's `time_modified` if the + // project id and name match, even if the snapshot ID does + // not match. diesel supports adding a filter below like so + // (marked with >>): + // + // .on_conflict((dsl::project_id, dsl::name)) + // .filter_target(dsl::time_deleted.is_null()) + // .do_update() + // .set(dsl::time_modified.eq(dsl::time_modified)) + // >> .filter(dsl::id.eq(snapshot.id())) + // + // which will restrict the `insert_into`'s set so that it + // only applies if the snapshot ID matches. But, + // AsyncInsertError does not have a ObjectAlreadyExists + // variant, so this will be returned as CollectionNotFound + // due to the `insert_into` failing. + // + // If this function is passed a snapshot with an ID that + // does not match, but a project and name that does, return + // ObjectAlreadyExists here. - if let Some(existing_snapshot_id) = existing_snapshot_id { - if existing_snapshot_id != snapshot.id() { - return Err(TransactionError::CustomError( - CustomError::ResourceAlreadyExists, - )); - } - } + let existing_snapshot_id: Option = dsl::snapshot + .filter(dsl::time_deleted.is_null()) + .filter(dsl::name.eq(snapshot.name().to_string())) + .filter(dsl::project_id.eq(snapshot.project_id)) + .select(dsl::id) + .limit(1) + .first_async(&conn) + .await + .optional()?; - Project::insert_resource( - project_id, - diesel::insert_into(dsl::snapshot) - .values(snapshot) - .on_conflict((dsl::project_id, dsl::name)) - .filter_target(dsl::time_deleted.is_null()) - .do_update() - .set(dsl::time_modified.eq(dsl::time_modified)), - ) - .insert_and_get_result_async(&conn) - .await - .map_err(|e| { - TransactionError::CustomError(CustomError::InsertError(e)) - }) - }) - .await - .map_err(|e: TxnError| match e { - TxnError::CustomError(e) => match e { - CustomError::ResourceAlreadyExists => { - Error::ObjectAlreadyExists { - type_name: ResourceType::Snapshot, - object_name: snapshot_name, + if let Some(existing_snapshot_id) = existing_snapshot_id { + if existing_snapshot_id != snapshot.id() { + return Err(err.bail(Error::ObjectAlreadyExists { + type_name: ResourceType::Snapshot, + object_name: snapshot_name, + })); } } - CustomError::InsertError(e) => match e { + + Project::insert_resource( + project_id, + diesel::insert_into(dsl::snapshot) + .values(snapshot) + .on_conflict((dsl::project_id, dsl::name)) + .filter_target(dsl::time_deleted.is_null()) + .do_update() + .set(dsl::time_modified.eq(dsl::time_modified)), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { AsyncInsertError::CollectionNotFound => { - Error::ObjectNotFound { + err.bail(Error::ObjectNotFound { type_name: ResourceType::Project, lookup_type: LookupType::ById(project_id), - } - } - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + }) } - }, - }, - TxnError::Database(e) => { + AsyncInsertError::DatabaseError(e) => e, + }) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + err + } else { public_error_from_diesel(e, ErrorHandler::Server) } })?; diff --git a/nexus/db-queries/src/db/datastore/switch_interface.rs b/nexus/db-queries/src/db/datastore/switch_interface.rs index 88cff50471..67f16fa08f 100644 --- a/nexus/db-queries/src/db/datastore/switch_interface.rs +++ b/nexus/db-queries/src/db/datastore/switch_interface.rs @@ -11,11 +11,10 @@ use crate::db::datastore::address_lot::{ }; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::LoopbackAddress; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; -use diesel::result::Error as DieselError; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use ipnetwork::IpNetwork; use nexus_types::external_api::params::LoopbackAddressCreate; @@ -40,80 +39,78 @@ impl DataStore { ReserveBlock(ReserveBlockError), } - type TxnError = TransactionError; - let conn = self.pool_connection_authorized(opctx).await?; let inet = IpNetwork::new(params.address, params.mask) .map_err(|_| Error::invalid_request("invalid address"))?; + let err = OptionalError::new(); + // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - let lot_id = authz_address_lot.id(); - let (block, rsvd_block) = - crate::db::datastore::address_lot::try_reserve_block( - lot_id, - inet.ip().into(), - params.anycast, - &conn, - ) - .await - .map_err(|e| match e { - ReserveBlockTxnError::CustomError(err) => { - TxnError::CustomError( - LoopbackAddressCreateError::ReserveBlock(err), + self.transaction_retry_wrapper("loopback_address_create") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let lot_id = authz_address_lot.id(); + let (block, rsvd_block) = + crate::db::datastore::address_lot::try_reserve_block( + lot_id, + inet.ip().into(), + params.anycast, + &conn, ) + .await + .map_err(|e| match e { + ReserveBlockTxnError::CustomError(e) => err.bail( + LoopbackAddressCreateError::ReserveBlock(e), + ), + ReserveBlockTxnError::Database(e) => e, + })?; + + // Address block reserved, now create the loopback address. + + let addr = LoopbackAddress::new( + id, + block.id, + rsvd_block.id, + params.rack_id, + params.switch_location.to_string(), + inet, + params.anycast, + ); + + let db_addr: LoopbackAddress = + diesel::insert_into(dsl::loopback_address) + .values(addr) + .returning(LoopbackAddress::as_returning()) + .get_result_async(&conn) + .await?; + + Ok(db_addr) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + LoopbackAddressCreateError::ReserveBlock( + ReserveBlockError::AddressUnavailable, + ) => Error::invalid_request("address unavailable"), + LoopbackAddressCreateError::ReserveBlock( + ReserveBlockError::AddressNotInLot, + ) => Error::invalid_request("address not in lot"), } - ReserveBlockTxnError::Database(err) => { - TxnError::Database(err) - } - })?; - - // Address block reserved, now create the loopback address. - - let addr = LoopbackAddress::new( - id, - block.id, - rsvd_block.id, - params.rack_id, - params.switch_location.to_string(), - inet, - params.anycast, - ); - - let db_addr: LoopbackAddress = - diesel::insert_into(dsl::loopback_address) - .values(addr) - .returning(LoopbackAddress::as_returning()) - .get_result_async(&conn) - .await?; - - Ok(db_addr) - }) - .await - .map_err(|e| match e { - TxnError::CustomError( - LoopbackAddressCreateError::ReserveBlock( - ReserveBlockError::AddressUnavailable, - ), - ) => Error::invalid_request("address unavailable"), - TxnError::CustomError( - LoopbackAddressCreateError::ReserveBlock( - ReserveBlockError::AddressNotInLot, - ), - ) => Error::invalid_request("address not in lot"), - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::LoopbackAddress, - &format!("lo {}", inet), - ), - ), - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, - }) + } else { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::LoopbackAddress, + &format!("lo {}", inet), + ), + ) + } + }) } pub async fn loopback_address_delete( @@ -130,22 +127,23 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - let la = diesel::delete(dsl::loopback_address) - .filter(dsl::id.eq(id)) - .returning(LoopbackAddress::as_returning()) - .get_result_async(&conn) - .await?; - - diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) - .filter(rsvd_block_dsl::id.eq(la.rsvd_address_lot_block_id)) - .execute_async(&conn) - .await?; - - Ok(()) - }) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + self.transaction_retry_wrapper("loopback_address_delete") + .transaction(&conn, |conn| async move { + let la = diesel::delete(dsl::loopback_address) + .filter(dsl::id.eq(id)) + .returning(LoopbackAddress::as_returning()) + .get_result_async(&conn) + .await?; + + diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) + .filter(rsvd_block_dsl::id.eq(la.rsvd_address_lot_block_id)) + .execute_async(&conn) + .await?; + + Ok(()) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn loopback_address_get( diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index 6bd4e61f70..221feee23c 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -11,7 +11,6 @@ use crate::db::datastore::address_lot::{ use crate::db::datastore::UpdatePrecondition; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::{ LldpServiceConfig, Name, SwitchInterfaceConfig, SwitchPort, SwitchPortAddressConfig, SwitchPortBgpPeerConfig, SwitchPortConfig, @@ -20,8 +19,8 @@ use crate::db::model::{ SwitchVlanInterfaceConfig, }; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; -use diesel::result::Error as DieselError; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::{ CombineDsl, ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl, SelectableHelper, @@ -163,283 +162,285 @@ impl DataStore { BgpConfigNotFound, ReserveBlock(ReserveBlockError), } - type TxnError = TransactionError; type SpsCreateError = SwitchPortSettingsCreateError; + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - // create the top level port settings object - let port_settings = match id { - Some(id) => SwitchPortSettings::with_id(id, ¶ms.identity), - None => SwitchPortSettings::new(¶ms.identity), - }; - //let port_settings = SwitchPortSettings::new(¶ms.identity); - let db_port_settings: SwitchPortSettings = - diesel::insert_into(port_settings_dsl::switch_port_settings) - .values(port_settings) - .returning(SwitchPortSettings::as_returning()) - .get_result_async(&conn) - .await?; - - let psid = db_port_settings.identity.id; - - // add the port config - let port_config = SwitchPortConfig::new( - psid, - params.port_config.geometry.into(), - ); - - let db_port_config: SwitchPortConfig = - diesel::insert_into(port_config_dsl::switch_port_settings_port_config) - .values(port_config) - .returning(SwitchPortConfig::as_returning()) - .get_result_async(&conn) - .await?; - - let mut result = SwitchPortSettingsCombinedResult{ - settings: db_port_settings, - groups: Vec::new(), - port: db_port_config, - links: Vec::new(), - link_lldp: Vec::new(), - interfaces: Vec::new(), - vlan_interfaces: Vec::new(), - routes: Vec::new(), - bgp_peers: Vec::new(), - addresses: Vec::new(), - }; - - //TODO validate link configs consistent with port geometry. - // - https://github.com/oxidecomputer/omicron/issues/2816 - - let mut lldp_config = Vec::with_capacity(params.links.len()); - let mut link_config = Vec::with_capacity(params.links.len()); - - for (link_name, c) in ¶ms.links { - let lldp_config_id = match c.lldp.lldp_config { - Some(_) => todo!(), // TODO actual lldp support - None => None, - }; - let lldp_svc_config = - LldpServiceConfig::new(c.lldp.enabled, lldp_config_id); - - lldp_config.push(lldp_svc_config.clone()); - link_config.push(SwitchPortLinkConfig::new( - psid, - lldp_svc_config.id, - link_name.clone(), - c.mtu, - c.fec.into(), - c.speed.into(), - c.autoneg, - )); - } - result.link_lldp = - diesel::insert_into(lldp_config_dsl::lldp_service_config) - .values(lldp_config.clone()) - .returning(LldpServiceConfig::as_returning()) - .get_results_async(&conn) - .await?; - result.links = - diesel::insert_into( - link_config_dsl::switch_port_settings_link_config) - .values(link_config) - .returning(SwitchPortLinkConfig::as_returning()) - .get_results_async(&conn) - .await?; - - let mut interface_config = Vec::with_capacity(params.interfaces.len()); - let mut vlan_interface_config = Vec::new(); - for (interface_name, i) in ¶ms.interfaces { - let ifx_config = SwitchInterfaceConfig::new( - psid, - interface_name.clone(), - i.v6_enabled, - i.kind.into(), - ); - interface_config.push(ifx_config.clone()); - if let params::SwitchInterfaceKind::Vlan(vlan_if) = i.kind { - vlan_interface_config.push(SwitchVlanInterfaceConfig::new( - ifx_config.id, - vlan_if.vid, - )); - } - } - result.interfaces = - diesel::insert_into( - interface_config_dsl::switch_port_settings_interface_config) - .values(interface_config) - .returning(SwitchInterfaceConfig::as_returning()) - .get_results_async(&conn) - .await?; - result.vlan_interfaces = - diesel::insert_into(vlan_config_dsl::switch_vlan_interface_config) - .values(vlan_interface_config) - .returning(SwitchVlanInterfaceConfig::as_returning()) - .get_results_async(&conn) - .await?; - - - let mut route_config = Vec::with_capacity(params.routes.len()); - - for (interface_name, r) in ¶ms.routes { - for route in &r.routes { - route_config.push(SwitchPortRouteConfig::new( - psid, - interface_name.clone(), - route.dst.into(), - route.gw.into(), - route.vid.map(Into::into), - )); - } - } - result.routes = - diesel::insert_into( - route_config_dsl::switch_port_settings_route_config) - .values(route_config) - .returning(SwitchPortRouteConfig::as_returning()) - .get_results_async(&conn) - .await?; - - let mut bgp_peer_config = Vec::new(); - for (interface_name, peer_config) in ¶ms.bgp_peers { - for p in &peer_config.peers { - use db::schema::bgp_config; - let bgp_config_id = match &p.bgp_config { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - bgp_config_dsl::bgp_config - .filter(bgp_config::time_deleted.is_null()) - .filter(bgp_config::name.eq(name)) - .select(bgp_config::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpConfigNotFound, - ) - )? - } + self.transaction_retry_wrapper("switch_port_settings_create") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + // create the top level port settings object + let port_settings = match id { + Some(id) => SwitchPortSettings::with_id(id, ¶ms.identity), + None => SwitchPortSettings::new(¶ms.identity), }; - - bgp_peer_config.push(SwitchPortBgpPeerConfig::new( + //let port_settings = SwitchPortSettings::new(¶ms.identity); + let db_port_settings: SwitchPortSettings = + diesel::insert_into(port_settings_dsl::switch_port_settings) + .values(port_settings) + .returning(SwitchPortSettings::as_returning()) + .get_result_async(&conn) + .await?; + + let psid = db_port_settings.identity.id; + + // add the port config + let port_config = SwitchPortConfig::new( psid, - bgp_config_id, - interface_name.clone(), - p.addr.into(), - p.hold_time.into(), - p.idle_hold_time.into(), - p.delay_open.into(), - p.connect_retry.into(), - p.keepalive.into(), - )); + params.port_config.geometry.into(), + ); + + let db_port_config: SwitchPortConfig = + diesel::insert_into(port_config_dsl::switch_port_settings_port_config) + .values(port_config) + .returning(SwitchPortConfig::as_returning()) + .get_result_async(&conn) + .await?; + + let mut result = SwitchPortSettingsCombinedResult{ + settings: db_port_settings, + groups: Vec::new(), + port: db_port_config, + links: Vec::new(), + link_lldp: Vec::new(), + interfaces: Vec::new(), + vlan_interfaces: Vec::new(), + routes: Vec::new(), + bgp_peers: Vec::new(), + addresses: Vec::new(), + }; - } - } - result.bgp_peers = - diesel::insert_into( - bgp_peer_dsl::switch_port_settings_bgp_peer_config) - .values(bgp_peer_config) - .returning(SwitchPortBgpPeerConfig::as_returning()) - .get_results_async(&conn) - .await?; + //TODO validate link configs consistent with port geometry. + // - https://github.com/oxidecomputer/omicron/issues/2816 + + let mut lldp_config = Vec::with_capacity(params.links.len()); + let mut link_config = Vec::with_capacity(params.links.len()); + + for (link_name, c) in ¶ms.links { + let lldp_config_id = match c.lldp.lldp_config { + Some(_) => todo!(), // TODO actual lldp support + None => None, + }; + let lldp_svc_config = + LldpServiceConfig::new(c.lldp.enabled, lldp_config_id); + + lldp_config.push(lldp_svc_config.clone()); + link_config.push(SwitchPortLinkConfig::new( + psid, + lldp_svc_config.id, + link_name.clone(), + c.mtu, + c.fec.into(), + c.speed.into(), + c.autoneg, + )); + } + result.link_lldp = + diesel::insert_into(lldp_config_dsl::lldp_service_config) + .values(lldp_config.clone()) + .returning(LldpServiceConfig::as_returning()) + .get_results_async(&conn) + .await?; + result.links = + diesel::insert_into( + link_config_dsl::switch_port_settings_link_config) + .values(link_config) + .returning(SwitchPortLinkConfig::as_returning()) + .get_results_async(&conn) + .await?; + + let mut interface_config = Vec::with_capacity(params.interfaces.len()); + let mut vlan_interface_config = Vec::new(); + for (interface_name, i) in ¶ms.interfaces { + let ifx_config = SwitchInterfaceConfig::new( + psid, + interface_name.clone(), + i.v6_enabled, + i.kind.into(), + ); + interface_config.push(ifx_config.clone()); + if let params::SwitchInterfaceKind::Vlan(vlan_if) = i.kind { + vlan_interface_config.push(SwitchVlanInterfaceConfig::new( + ifx_config.id, + vlan_if.vid, + )); + } + } + result.interfaces = + diesel::insert_into( + interface_config_dsl::switch_port_settings_interface_config) + .values(interface_config) + .returning(SwitchInterfaceConfig::as_returning()) + .get_results_async(&conn) + .await?; + result.vlan_interfaces = + diesel::insert_into(vlan_config_dsl::switch_vlan_interface_config) + .values(vlan_interface_config) + .returning(SwitchVlanInterfaceConfig::as_returning()) + .get_results_async(&conn) + .await?; + + let mut route_config = Vec::with_capacity(params.routes.len()); + + for (interface_name, r) in ¶ms.routes { + for route in &r.routes { + route_config.push(SwitchPortRouteConfig::new( + psid, + interface_name.clone(), + route.dst.into(), + route.gw.into(), + route.vid.map(Into::into), + )); + } + } + result.routes = + diesel::insert_into( + route_config_dsl::switch_port_settings_route_config) + .values(route_config) + .returning(SwitchPortRouteConfig::as_returning()) + .get_results_async(&conn) + .await?; + + let mut bgp_peer_config = Vec::new(); + for (interface_name, peer_config) in ¶ms.bgp_peers { + for p in &peer_config.peers { + use db::schema::bgp_config; + let bgp_config_id = match &p.bgp_config { + NameOrId::Id(id) => *id, + NameOrId::Name(name) => { + let name = name.to_string(); + bgp_config_dsl::bgp_config + .filter(bgp_config::time_deleted.is_null()) + .filter(bgp_config::name.eq(name)) + .select(bgp_config::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or( + diesel_error, + SwitchPortSettingsCreateError::BgpConfigNotFound + ) + })? + } + }; + + bgp_peer_config.push(SwitchPortBgpPeerConfig::new( + psid, + bgp_config_id, + interface_name.clone(), + p.addr.into(), + p.hold_time.into(), + p.idle_hold_time.into(), + p.delay_open.into(), + p.connect_retry.into(), + p.keepalive.into(), + )); - let mut address_config = Vec::new(); - use db::schema::address_lot; - for (interface_name, a) in ¶ms.addresses { - for address in &a.addresses { - let address_lot_id = match &address.address_lot { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - address_lot_dsl::address_lot - .filter(address_lot::time_deleted.is_null()) - .filter(address_lot::name.eq(name)) - .select(address_lot::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| - TxnError::CustomError( - SwitchPortSettingsCreateError::AddressLotNotFound, - ) - )? } - }; - // TODO: Reduce DB round trips needed for reserving ip blocks - // https://github.com/oxidecomputer/omicron/issues/3060 - let (block, rsvd_block) = - crate::db::datastore::address_lot::try_reserve_block( - address_lot_id, - address.address.ip().into(), - // TODO: Should we allow anycast addresses for switch_ports? - // anycast - false, - &conn - ) - .await - .map_err(|e| match e { - ReserveBlockTxnError::CustomError(err) => { - TxnError::CustomError( - SwitchPortSettingsCreateError::ReserveBlock(err) + } + result.bgp_peers = + diesel::insert_into( + bgp_peer_dsl::switch_port_settings_bgp_peer_config) + .values(bgp_peer_config) + .returning(SwitchPortBgpPeerConfig::as_returning()) + .get_results_async(&conn) + .await?; + + let mut address_config = Vec::new(); + use db::schema::address_lot; + for (interface_name, a) in ¶ms.addresses { + for address in &a.addresses { + let address_lot_id = match &address.address_lot { + NameOrId::Id(id) => *id, + NameOrId::Name(name) => { + let name = name.to_string(); + address_lot_dsl::address_lot + .filter(address_lot::time_deleted.is_null()) + .filter(address_lot::name.eq(name)) + .select(address_lot::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or( + diesel_error, + SwitchPortSettingsCreateError::AddressLotNotFound + ) + })? + } + }; + // TODO: Reduce DB round trips needed for reserving ip blocks + // https://github.com/oxidecomputer/omicron/issues/3060 + let (block, rsvd_block) = + crate::db::datastore::address_lot::try_reserve_block( + address_lot_id, + address.address.ip().into(), + // TODO: Should we allow anycast addresses for switch_ports? + // anycast + false, + &conn ) - } - ReserveBlockTxnError::Database(err) => TxnError::Database(err), - })?; - - address_config.push(SwitchPortAddressConfig::new( - psid, - block.id, - rsvd_block.id, - address.address.into(), - interface_name.clone(), - )); + .await + .map_err(|e| match e { + ReserveBlockTxnError::CustomError(e) => { + err.bail(SwitchPortSettingsCreateError::ReserveBlock(e)) + } + ReserveBlockTxnError::Database(e) => e, + })?; + + address_config.push(SwitchPortAddressConfig::new( + psid, + block.id, + rsvd_block.id, + address.address.into(), + interface_name.clone(), + )); + } + } + result.addresses = + diesel::insert_into( + address_config_dsl::switch_port_settings_address_config) + .values(address_config) + .returning(SwitchPortAddressConfig::as_returning()) + .get_results_async(&conn) + .await?; + + Ok(result) } } - result.addresses = - diesel::insert_into( - address_config_dsl::switch_port_settings_address_config) - .values(address_config) - .returning(SwitchPortAddressConfig::as_returning()) - .get_results_async(&conn) - .await?; - - Ok(result) - }) + ) .await - .map_err(|e| match e { - TxnError::CustomError(SpsCreateError::AddressLotNotFound) => { - Error::invalid_request("AddressLot not found") - } - TxnError::CustomError(SpsCreateError::BgpConfigNotFound) => { - Error::invalid_request("BGP config not found") - } - TxnError::CustomError( - SwitchPortSettingsCreateError::ReserveBlock( - ReserveBlockError::AddressUnavailable - ) - ) => Error::invalid_request("address unavailable"), - TxnError::CustomError( - SwitchPortSettingsCreateError::ReserveBlock( - ReserveBlockError::AddressNotInLot - ) - ) => Error::invalid_request("address not in lot"), - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => public_error_from_diesel( + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SpsCreateError::AddressLotNotFound => { + Error::invalid_request("AddressLot not found") + } + SpsCreateError::BgpConfigNotFound => { + Error::invalid_request("BGP config not found") + } + SwitchPortSettingsCreateError::ReserveBlock( + ReserveBlockError::AddressUnavailable + ) => Error::invalid_request("address unavailable"), + SwitchPortSettingsCreateError::ReserveBlock( + ReserveBlockError::AddressNotInLot + ) => Error::invalid_request("address not in lot"), + } + } else { + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SwitchPortSettings, params.identity.name.as_str(), ), - ), - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, + ) + } }) } @@ -454,7 +455,6 @@ impl DataStore { enum SwitchPortSettingsDeleteError { SwitchPortSettingsNotFound, } - type TxnError = TransactionError; let conn = self.pool_connection_authorized(opctx).await?; @@ -463,173 +463,178 @@ impl DataStore { Some(name_or_id) => name_or_id, }; + let err = OptionalError::new(); + // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - - use db::schema::switch_port_settings; - let id = match selector { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - port_settings_dsl::switch_port_settings - .filter(switch_port_settings::time_deleted.is_null()) - .filter(switch_port_settings::name.eq(name)) - .select(switch_port_settings::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| - TxnError::CustomError( - SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound, - ) - )? - } - }; + self.transaction_retry_wrapper("switch_port_settings_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::switch_port_settings; + let id = match selector { + NameOrId::Id(id) => *id, + NameOrId::Name(name) => { + let name = name.to_string(); + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::name.eq(name)) + .select(switch_port_settings::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or( + diesel_error, + SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound + ) + })? + } + }; - // delete the top level port settings object - diesel::delete(port_settings_dsl::switch_port_settings) - .filter(switch_port_settings::id.eq(id)) - .execute_async(&conn) - .await?; + // delete the top level port settings object + diesel::delete(port_settings_dsl::switch_port_settings) + .filter(switch_port_settings::id.eq(id)) + .execute_async(&conn) + .await?; - // delete the port config object - use db::schema::switch_port_settings_port_config::{ - self as sps_port_config, dsl as port_config_dsl, - }; - diesel::delete(port_config_dsl::switch_port_settings_port_config) - .filter(sps_port_config::port_settings_id.eq(id)) - .execute_async(&conn) - .await?; + // delete the port config object + use db::schema::switch_port_settings_port_config::{ + self as sps_port_config, dsl as port_config_dsl, + }; + diesel::delete(port_config_dsl::switch_port_settings_port_config) + .filter(sps_port_config::port_settings_id.eq(id)) + .execute_async(&conn) + .await?; - // delete the link configs - use db::schema::switch_port_settings_link_config::{ - self as sps_link_config, dsl as link_config_dsl, - }; - let links: Vec = - diesel::delete( - link_config_dsl::switch_port_settings_link_config - ) - .filter( - sps_link_config::port_settings_id.eq(id) - ) - .returning(SwitchPortLinkConfig::as_returning()) - .get_results_async(&conn) - .await?; + // delete the link configs + use db::schema::switch_port_settings_link_config::{ + self as sps_link_config, dsl as link_config_dsl, + }; + let links: Vec = + diesel::delete( + link_config_dsl::switch_port_settings_link_config + ) + .filter( + sps_link_config::port_settings_id.eq(id) + ) + .returning(SwitchPortLinkConfig::as_returning()) + .get_results_async(&conn) + .await?; - // delete lldp configs - use db::schema::lldp_service_config::{self, dsl as lldp_config_dsl}; - let lldp_svc_ids: Vec = links - .iter() - .map(|link| link.lldp_service_config_id) - .collect(); - diesel::delete(lldp_config_dsl::lldp_service_config) - .filter(lldp_service_config::id.eq_any(lldp_svc_ids)) - .execute_async(&conn) - .await?; + // delete lldp configs + use db::schema::lldp_service_config::{self, dsl as lldp_config_dsl}; + let lldp_svc_ids: Vec = links + .iter() + .map(|link| link.lldp_service_config_id) + .collect(); + diesel::delete(lldp_config_dsl::lldp_service_config) + .filter(lldp_service_config::id.eq_any(lldp_svc_ids)) + .execute_async(&conn) + .await?; - // delete interface configs - use db::schema::switch_port_settings_interface_config::{ - self as sps_interface_config, dsl as interface_config_dsl, - }; + // delete interface configs + use db::schema::switch_port_settings_interface_config::{ + self as sps_interface_config, dsl as interface_config_dsl, + }; - let interfaces: Vec = - diesel::delete( - interface_config_dsl::switch_port_settings_interface_config - ) - .filter( - sps_interface_config::port_settings_id.eq(id) - ) - .returning(SwitchInterfaceConfig::as_returning()) - .get_results_async(&conn) - .await?; + let interfaces: Vec = + diesel::delete( + interface_config_dsl::switch_port_settings_interface_config + ) + .filter( + sps_interface_config::port_settings_id.eq(id) + ) + .returning(SwitchInterfaceConfig::as_returning()) + .get_results_async(&conn) + .await?; - // delete any vlan interfaces - use db::schema::switch_vlan_interface_config::{ - self, dsl as vlan_config_dsl, - }; - let interface_ids: Vec = interfaces - .iter() - .map(|interface| interface.id) - .collect(); - - diesel::delete(vlan_config_dsl::switch_vlan_interface_config) - .filter( - switch_vlan_interface_config::interface_config_id.eq_any( - interface_ids + // delete any vlan interfaces + use db::schema::switch_vlan_interface_config::{ + self, dsl as vlan_config_dsl, + }; + let interface_ids: Vec = interfaces + .iter() + .map(|interface| interface.id) + .collect(); + + diesel::delete(vlan_config_dsl::switch_vlan_interface_config) + .filter( + switch_vlan_interface_config::interface_config_id.eq_any( + interface_ids + ) ) + .execute_async(&conn) + .await?; + + // delete route configs + use db::schema::switch_port_settings_route_config; + use db::schema::switch_port_settings_route_config::dsl + as route_config_dsl; + + diesel::delete( + route_config_dsl::switch_port_settings_route_config ) + .filter(switch_port_settings_route_config::port_settings_id.eq(id)) .execute_async(&conn) .await?; - // delete route configs - use db::schema::switch_port_settings_route_config; - use db::schema::switch_port_settings_route_config::dsl - as route_config_dsl; + // delete bgp configurations + use db::schema::switch_port_settings_bgp_peer_config as bgp_peer; + use db::schema::switch_port_settings_bgp_peer_config::dsl + as bgp_peer_dsl; - diesel::delete( - route_config_dsl::switch_port_settings_route_config - ) - .filter(switch_port_settings_route_config::port_settings_id.eq(id)) - .execute_async(&conn) - .await?; + diesel::delete(bgp_peer_dsl::switch_port_settings_bgp_peer_config) + .filter(bgp_peer::port_settings_id.eq(id)) + .execute_async(&conn) + .await?; - // delete bgp configurations - use db::schema::switch_port_settings_bgp_peer_config as bgp_peer; - use db::schema::switch_port_settings_bgp_peer_config::dsl - as bgp_peer_dsl; + // delete address configs + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; - diesel::delete(bgp_peer_dsl::switch_port_settings_bgp_peer_config) - .filter(bgp_peer::port_settings_id.eq(id)) - .execute_async(&conn) + let port_settings_addrs = diesel::delete( + address_config_dsl::switch_port_settings_address_config, + ) + .filter(address_config::port_settings_id.eq(id)) + .returning(SwitchPortAddressConfig::as_returning()) + .get_results_async(&conn) .await?; - // delete address configs - use db::schema::switch_port_settings_address_config::{ - self as address_config, dsl as address_config_dsl, - }; - - let port_settings_addrs = diesel::delete( - address_config_dsl::switch_port_settings_address_config, - ) - .filter(address_config::port_settings_id.eq(id)) - .returning(SwitchPortAddressConfig::as_returning()) - .get_results_async(&conn) - .await?; + use db::schema::address_lot_rsvd_block::dsl as rsvd_block_dsl; - use db::schema::address_lot_rsvd_block::dsl as rsvd_block_dsl; + for ps in &port_settings_addrs { + diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) + .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) + .execute_async(&conn) + .await?; + } - for ps in &port_settings_addrs { - diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) - .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) - .execute_async(&conn) - .await?; + Ok(()) } - - Ok(()) }) .await - .map_err(|e| match e { - TxnError::CustomError( - SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound) => { - Error::invalid_request("port settings not found") + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound => { + Error::invalid_request("port settings not found") + } + } + } else { + let name = match ¶ms.port_settings { + Some(name_or_id) => name_or_id.to_string(), + None => String::new(), + }; + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::SwitchPortSettings, + &name, + ), + ) } - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => { - let name = match ¶ms.port_settings { - Some(name_or_id) => name_or_id.to_string(), - None => String::new(), - }; - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::SwitchPortSettings, - &name, - ), - ) - }, - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, }) } @@ -666,174 +671,178 @@ impl DataStore { enum SwitchPortSettingsGetError { NotFound(external::Name), } - type TxnError = TransactionError; + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - // get the top level port settings object - use db::schema::switch_port_settings::{ - self, dsl as port_settings_dsl, - }; - - let id = match name_or_id { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name_str = name.to_string(); + self.transaction_retry_wrapper("switch_port_settings_get") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + // get the top level port settings object + use db::schema::switch_port_settings::{ + self, dsl as port_settings_dsl, + }; + + let id = match name_or_id { + NameOrId::Id(id) => *id, + NameOrId::Name(name) => { + let name_str = name.to_string(); + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::name.eq(name_str)) + .select(switch_port_settings::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or_else(diesel_error, |_| { + SwitchPortSettingsGetError::NotFound( + name.clone(), + ) + }) + })? + } + }; + + let settings: SwitchPortSettings = port_settings_dsl::switch_port_settings .filter(switch_port_settings::time_deleted.is_null()) - .filter(switch_port_settings::name.eq(name_str)) - .select(switch_port_settings::id) + .filter(switch_port_settings::id.eq(id)) + .select(SwitchPortSettings::as_select()) .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError( - SwitchPortSettingsGetError::NotFound( - name.clone(), - ), - ) - })? - } - }; - - let settings: SwitchPortSettings = - port_settings_dsl::switch_port_settings - .filter(switch_port_settings::time_deleted.is_null()) - .filter(switch_port_settings::id.eq(id)) - .select(SwitchPortSettings::as_select()) - .limit(1) - .first_async::(&conn) - .await?; + .first_async::(&conn) + .await?; - // get the port config - use db::schema::switch_port_settings_port_config::{ - self as port_config, dsl as port_config_dsl, - }; - let port: SwitchPortConfig = - port_config_dsl::switch_port_settings_port_config - .filter(port_config::port_settings_id.eq(id)) - .select(SwitchPortConfig::as_select()) - .limit(1) - .first_async::(&conn) - .await?; + // get the port config + use db::schema::switch_port_settings_port_config::{ + self as port_config, dsl as port_config_dsl, + }; + let port: SwitchPortConfig = + port_config_dsl::switch_port_settings_port_config + .filter(port_config::port_settings_id.eq(id)) + .select(SwitchPortConfig::as_select()) + .limit(1) + .first_async::(&conn) + .await?; - // initialize result - let mut result = - SwitchPortSettingsCombinedResult::new(settings, port); + // initialize result + let mut result = + SwitchPortSettingsCombinedResult::new(settings, port); - // get the link configs - use db::schema::switch_port_settings_link_config::{ - self as link_config, dsl as link_config_dsl, - }; + // get the link configs + use db::schema::switch_port_settings_link_config::{ + self as link_config, dsl as link_config_dsl, + }; - result.links = link_config_dsl::switch_port_settings_link_config - .filter(link_config::port_settings_id.eq(id)) - .select(SwitchPortLinkConfig::as_select()) - .load_async::(&conn) - .await?; + result.links = link_config_dsl::switch_port_settings_link_config + .filter(link_config::port_settings_id.eq(id)) + .select(SwitchPortLinkConfig::as_select()) + .load_async::(&conn) + .await?; - let lldp_svc_ids: Vec = result - .links - .iter() - .map(|link| link.lldp_service_config_id) - .collect(); - - use db::schema::lldp_service_config as lldp_config; - use db::schema::lldp_service_config::dsl as lldp_dsl; - result.link_lldp = lldp_dsl::lldp_service_config - .filter(lldp_config::id.eq_any(lldp_svc_ids)) - .select(LldpServiceConfig::as_select()) - .limit(1) - .load_async::(&conn) - .await?; + let lldp_svc_ids: Vec = result + .links + .iter() + .map(|link| link.lldp_service_config_id) + .collect(); + + use db::schema::lldp_service_config as lldp_config; + use db::schema::lldp_service_config::dsl as lldp_dsl; + result.link_lldp = lldp_dsl::lldp_service_config + .filter(lldp_config::id.eq_any(lldp_svc_ids)) + .select(LldpServiceConfig::as_select()) + .limit(1) + .load_async::(&conn) + .await?; - // get the interface configs - use db::schema::switch_port_settings_interface_config::{ - self as interface_config, dsl as interface_config_dsl, - }; + // get the interface configs + use db::schema::switch_port_settings_interface_config::{ + self as interface_config, dsl as interface_config_dsl, + }; - result.interfaces = - interface_config_dsl::switch_port_settings_interface_config - .filter(interface_config::port_settings_id.eq(id)) - .select(SwitchInterfaceConfig::as_select()) - .load_async::(&conn) + result.interfaces = + interface_config_dsl::switch_port_settings_interface_config + .filter(interface_config::port_settings_id.eq(id)) + .select(SwitchInterfaceConfig::as_select()) + .load_async::(&conn) + .await?; + + use db::schema::switch_vlan_interface_config as vlan_config; + use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; + let interface_ids: Vec = result + .interfaces + .iter() + .map(|interface| interface.id) + .collect(); + + result.vlan_interfaces = vlan_dsl::switch_vlan_interface_config + .filter(vlan_config::interface_config_id.eq_any(interface_ids)) + .select(SwitchVlanInterfaceConfig::as_select()) + .load_async::(&conn) .await?; - use db::schema::switch_vlan_interface_config as vlan_config; - use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; - let interface_ids: Vec = result - .interfaces - .iter() - .map(|interface| interface.id) - .collect(); - - result.vlan_interfaces = vlan_dsl::switch_vlan_interface_config - .filter(vlan_config::interface_config_id.eq_any(interface_ids)) - .select(SwitchVlanInterfaceConfig::as_select()) - .load_async::(&conn) - .await?; - - // get the route configs - use db::schema::switch_port_settings_route_config::{ - self as route_config, dsl as route_config_dsl, - }; + // get the route configs + use db::schema::switch_port_settings_route_config::{ + self as route_config, dsl as route_config_dsl, + }; - result.routes = route_config_dsl::switch_port_settings_route_config - .filter(route_config::port_settings_id.eq(id)) - .select(SwitchPortRouteConfig::as_select()) - .load_async::(&conn) - .await?; + result.routes = route_config_dsl::switch_port_settings_route_config + .filter(route_config::port_settings_id.eq(id)) + .select(SwitchPortRouteConfig::as_select()) + .load_async::(&conn) + .await?; - // get the bgp peer configs - use db::schema::switch_port_settings_bgp_peer_config::{ - self as bgp_peer, dsl as bgp_peer_dsl, - }; + // get the bgp peer configs + use db::schema::switch_port_settings_bgp_peer_config::{ + self as bgp_peer, dsl as bgp_peer_dsl, + }; - result.bgp_peers = - bgp_peer_dsl::switch_port_settings_bgp_peer_config - .filter(bgp_peer::port_settings_id.eq(id)) - .select(SwitchPortBgpPeerConfig::as_select()) - .load_async::(&conn) - .await?; + result.bgp_peers = + bgp_peer_dsl::switch_port_settings_bgp_peer_config + .filter(bgp_peer::port_settings_id.eq(id)) + .select(SwitchPortBgpPeerConfig::as_select()) + .load_async::(&conn) + .await?; - // get the address configs - use db::schema::switch_port_settings_address_config::{ - self as address_config, dsl as address_config_dsl, - }; + // get the address configs + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; - result.addresses = - address_config_dsl::switch_port_settings_address_config - .filter(address_config::port_settings_id.eq(id)) - .select(SwitchPortAddressConfig::as_select()) - .load_async::(&conn) - .await?; + result.addresses = + address_config_dsl::switch_port_settings_address_config + .filter(address_config::port_settings_id.eq(id)) + .select(SwitchPortAddressConfig::as_select()) + .load_async::(&conn) + .await?; - Ok(result) + Ok(result) + } }) .await - .map_err(|e| match e { - TxnError::CustomError(SwitchPortSettingsGetError::NotFound( - name, - )) => Error::not_found_by_name( - ResourceType::SwitchPortSettings, - &name, - ), - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => { - let name = name_or_id.to_string(); - public_error_from_diesel( - e, - ErrorHandler::Conflict( + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SwitchPortSettingsGetError::NotFound(name) => { + Error::not_found_by_name( ResourceType::SwitchPortSettings, &name, - ), - ) + ) + } } - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, + } else { + let name = name_or_id.to_string(); + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::SwitchPortSettings, + &name, + ), + ) + } }) } @@ -850,7 +859,8 @@ impl DataStore { enum SwitchPortCreateError { RackNotFound, } - type TxnError = TransactionError; + + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; let switch_port = SwitchPort::new( @@ -861,46 +871,59 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - use db::schema::rack; - use db::schema::rack::dsl as rack_dsl; - rack_dsl::rack - .filter(rack::id.eq(rack_id)) - .select(rack::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError(SwitchPortCreateError::RackNotFound) - })?; - - // insert switch port - use db::schema::switch_port::dsl as switch_port_dsl; - let db_switch_port: SwitchPort = - diesel::insert_into(switch_port_dsl::switch_port) - .values(switch_port) - .returning(SwitchPort::as_returning()) - .get_result_async(&conn) - .await?; + self.transaction_retry_wrapper("switch_port_create") + .transaction(&conn, |conn| { + let err = err.clone(); + let switch_port = switch_port.clone(); + async move { + use db::schema::rack; + use db::schema::rack::dsl as rack_dsl; + rack_dsl::rack + .filter(rack::id.eq(rack_id)) + .select(rack::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or( + e, + SwitchPortCreateError::RackNotFound, + ) + })?; - Ok(db_switch_port) - }) - .await - .map_err(|e| match e { - TxnError::CustomError(SwitchPortCreateError::RackNotFound) => { - Error::invalid_request("rack not found") - } - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::SwitchPort, - &format!("{}/{}/{}", rack_id, &switch_location, &port,), - ), - ), - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, - }) + // insert switch port + use db::schema::switch_port::dsl as switch_port_dsl; + let db_switch_port: SwitchPort = + diesel::insert_into(switch_port_dsl::switch_port) + .values(switch_port) + .returning(SwitchPort::as_returning()) + .get_result_async(&conn) + .await?; + + Ok(db_switch_port) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SwitchPortCreateError::RackNotFound => { + Error::invalid_request("rack not found") + } + } + } else { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::SwitchPort, + &format!( + "{}/{}/{}", + rack_id, &switch_location, &port, + ), + ), + ) + } + }) } pub async fn switch_port_delete( @@ -914,58 +937,75 @@ impl DataStore { NotFound, ActiveSettings, } - type TxnError = TransactionError; + + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - use db::schema::switch_port; - use db::schema::switch_port::dsl as switch_port_dsl; - - let switch_location = params.switch_location.to_string(); - let port_name = portname.to_string(); - let port: SwitchPort = switch_port_dsl::switch_port - .filter(switch_port::rack_id.eq(params.rack_id)) - .filter( - switch_port::switch_location.eq(switch_location.clone()), - ) - .filter(switch_port::port_name.eq(port_name.clone())) - .select(SwitchPort::as_select()) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError(SwitchPortDeleteError::NotFound) - })?; - - if port.port_settings_id.is_some() { - return Err(TxnError::CustomError( - SwitchPortDeleteError::ActiveSettings, - )); - } + self.transaction_retry_wrapper("switch_port_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::switch_port; + use db::schema::switch_port::dsl as switch_port_dsl; + + let switch_location = params.switch_location.to_string(); + let port_name = portname.to_string(); + let port: SwitchPort = switch_port_dsl::switch_port + .filter(switch_port::rack_id.eq(params.rack_id)) + .filter( + switch_port::switch_location + .eq(switch_location.clone()), + ) + .filter(switch_port::port_name.eq(port_name.clone())) + .select(SwitchPort::as_select()) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or( + diesel_error, + SwitchPortDeleteError::NotFound, + ) + })?; - diesel::delete(switch_port_dsl::switch_port) - .filter(switch_port::id.eq(port.id)) - .execute_async(&conn) - .await?; + if port.port_settings_id.is_some() { + return Err( + err.bail(SwitchPortDeleteError::ActiveSettings) + ); + } - Ok(()) - }) - .await - .map_err(|e| match e { - TxnError::CustomError(SwitchPortDeleteError::NotFound) => { - let name = &portname.clone(); - Error::not_found_by_name(ResourceType::SwitchPort, name) - } - TxnError::CustomError(SwitchPortDeleteError::ActiveSettings) => { - Error::invalid_request("must clear port settings first") - } - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + diesel::delete(switch_port_dsl::switch_port) + .filter(switch_port::id.eq(port.id)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SwitchPortDeleteError::NotFound => { + let name = &portname.clone(); + Error::not_found_by_name( + ResourceType::SwitchPort, + name, + ) + } + SwitchPortDeleteError::ActiveSettings => { + Error::invalid_request( + "must clear port settings first", + ) + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } pub async fn switch_port_list( diff --git a/nexus/db-queries/src/db/datastore/update.rs b/nexus/db-queries/src/db/datastore/update.rs index 8b1eecb781..0790bd458e 100644 --- a/nexus/db-queries/src/db/datastore/update.rs +++ b/nexus/db-queries/src/db/datastore/update.rs @@ -8,15 +8,13 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::{ - public_error_from_diesel, ErrorHandler, TransactionError, -}; +use crate::db::error::{public_error_from_diesel, ErrorHandler}; use crate::db::model::{ ComponentUpdate, SemverVersion, SystemUpdate, UpdateArtifact, UpdateDeployment, UpdateStatus, UpdateableComponent, }; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use nexus_db_model::SystemUpdateComponentUpdate; @@ -141,36 +139,40 @@ impl DataStore { let version_string = update.version.to_string(); - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - let db_update = diesel::insert_into(component_update::table) - .values(update.clone()) - .returning(ComponentUpdate::as_returning()) - .get_result_async(&conn) - .await?; - - diesel::insert_into(join_table::table) - .values(SystemUpdateComponentUpdate { - system_update_id, - component_update_id: update.id(), - }) - .returning(SystemUpdateComponentUpdate::as_returning()) - .get_result_async(&conn) - .await?; - - Ok(db_update) + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("create_component_update") + .transaction(&conn, |conn| { + let update = update.clone(); + async move { + let db_update = + diesel::insert_into(component_update::table) + .values(update.clone()) + .returning(ComponentUpdate::as_returning()) + .get_result_async(&conn) + .await?; + + diesel::insert_into(join_table::table) + .values(SystemUpdateComponentUpdate { + system_update_id, + component_update_id: update.id(), + }) + .returning(SystemUpdateComponentUpdate::as_returning()) + .get_result_async(&conn) + .await?; + + Ok(db_update) + } }) .await - .map_err(|e| match e { - TransactionError::CustomError(e) => e, - TransactionError::Database(e) => public_error_from_diesel( + .map_err(|e| { + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::ComponentUpdate, &version_string, ), - ), + ) }) } diff --git a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs index c5c2751723..230c3941ff 100644 --- a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs +++ b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs @@ -15,6 +15,7 @@ use crate::db::pool::DbConnection; use crate::db::queries::virtual_provisioning_collection_update::VirtualProvisioningCollectionUpdate; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; +use diesel::result::Error as DieselError; use omicron_common::api::external::{DeleteResult, Error}; use uuid::Uuid; @@ -52,13 +53,14 @@ impl DataStore { virtual_provisioning_collection, ) .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub(crate) async fn virtual_provisioning_collection_create_on_connection( &self, conn: &async_bb8_diesel::Connection, virtual_provisioning_collection: VirtualProvisioningCollection, - ) -> Result, Error> { + ) -> Result, DieselError> { use db::schema::virtual_provisioning_collection::dsl; let provisions: Vec = @@ -66,12 +68,10 @@ impl DataStore { .values(virtual_provisioning_collection) .on_conflict_do_nothing() .get_results_async(conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - })?; - self.virtual_provisioning_collection_producer - .append_all_metrics(&provisions)?; + .await?; + let _ = self + .virtual_provisioning_collection_producer + .append_all_metrics(&provisions); Ok(provisions) } @@ -103,16 +103,20 @@ impl DataStore { id: Uuid, ) -> DeleteResult { let conn = self.pool_connection_authorized(opctx).await?; - self.virtual_provisioning_collection_delete_on_connection(&conn, id) - .await + self.virtual_provisioning_collection_delete_on_connection( + &opctx.log, &conn, id, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Delete a [`VirtualProvisioningCollection`] object. pub(crate) async fn virtual_provisioning_collection_delete_on_connection( &self, + log: &slog::Logger, conn: &async_bb8_diesel::Connection, id: Uuid, - ) -> DeleteResult { + ) -> Result<(), DieselError> { use db::schema::virtual_provisioning_collection::dsl; // NOTE: We don't really need to extract the value we're deleting from @@ -122,13 +126,11 @@ impl DataStore { .filter(dsl::id.eq(id)) .returning(VirtualProvisioningCollection::as_select()) .get_result_async(conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .await?; if !collection.is_empty() { - return Err(Error::internal_error(&format!( - "Collection deleted while non-empty: {collection:?}" - ))); + warn!(log, "Collection deleted while non-empty: {collection:?}"); + return Err(DieselError::RollbackTransaction); } Ok(()) } diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index 5f126050ae..4f31efd610 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -8,15 +8,14 @@ use super::DataStore; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::identity::Asset; use crate::db::model::Dataset; use crate::db::model::Region; use crate::db::model::RegionSnapshot; use crate::db::model::Volume; use crate::db::queries::volume::DecreaseCrucibleResourceCountAndSoftDeleteVolume; +use crate::transaction_retry::OptionalError; use anyhow::bail; -use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use diesel::OptionalExtension; @@ -44,7 +43,6 @@ impl DataStore { #[error("Serde error during Volume creation: {0}")] SerdeError(#[from] serde_json::Error), } - type TxnError = TransactionError; // Grab all the targets that the volume construction request references. // Do this outside the transaction, as the data inside volume doesn't @@ -66,86 +64,91 @@ impl DataStore { crucible_targets }; - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - let maybe_volume: Option = dsl::volume - .filter(dsl::id.eq(volume.id())) - .select(Volume::as_select()) - .first_async(&conn) - .await - .optional() - .map_err(|e| { - TxnError::CustomError(VolumeCreationError::Public( - public_error_from_diesel(e, ErrorHandler::Server), - )) - })?; - - // If the volume existed already, return it and do not increase - // usage counts. - if let Some(volume) = maybe_volume { - return Ok(volume); - } - - // TODO do we need on_conflict do_nothing here? if the transaction - // model is read-committed, the SELECT above could return nothing, - // and the INSERT here could still result in a conflict. - // - // See also https://github.com/oxidecomputer/omicron/issues/1168 - let volume: Volume = diesel::insert_into(dsl::volume) - .values(volume.clone()) - .on_conflict(dsl::id) - .do_nothing() - .returning(Volume::as_returning()) - .get_result_async(&conn) - .await - .map_err(|e| { - TxnError::CustomError(VolumeCreationError::Public( - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Volume, - volume.id().to_string().as_str(), - ), - ), - )) - })?; + let err = OptionalError::new(); + let conn = self.pool_connection_unauthorized().await?; + self.transaction_retry_wrapper("volume_create") + .transaction(&conn, |conn| { + let err = err.clone(); + let crucible_targets = crucible_targets.clone(); + let volume = volume.clone(); + async move { + let maybe_volume: Option = dsl::volume + .filter(dsl::id.eq(volume.id())) + .select(Volume::as_select()) + .first_async(&conn) + .await + .optional()?; - // Increase the usage count for Crucible resources according to the - // contents of the volume. + // If the volume existed already, return it and do not increase + // usage counts. + if let Some(volume) = maybe_volume { + return Ok(volume); + } - // Increase the number of uses for each referenced region snapshot. - use db::schema::region_snapshot::dsl as rs_dsl; - for read_only_target in &crucible_targets.read_only_targets { - diesel::update(rs_dsl::region_snapshot) - .filter( - rs_dsl::snapshot_addr.eq(read_only_target.clone()), - ) - .filter(rs_dsl::deleting.eq(false)) - .set( - rs_dsl::volume_references - .eq(rs_dsl::volume_references + 1), - ) - .execute_async(&conn) + // TODO do we need on_conflict do_nothing here? if the transaction + // model is read-committed, the SELECT above could return nothing, + // and the INSERT here could still result in a conflict. + // + // See also https://github.com/oxidecomputer/omicron/issues/1168 + let volume: Volume = diesel::insert_into(dsl::volume) + .values(volume.clone()) + .on_conflict(dsl::id) + .do_nothing() + .returning(Volume::as_returning()) + .get_result_async(&conn) .await .map_err(|e| { - TxnError::CustomError(VolumeCreationError::Public( - public_error_from_diesel( - e, - ErrorHandler::Server, - ), - )) + err.bail_retryable_or_else(e, |e| { + VolumeCreationError::Public( + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Volume, + volume.id().to_string().as_str(), + ), + ), + ) + }) })?; - } - Ok(volume) + // Increase the usage count for Crucible resources according to the + // contents of the volume. + + // Increase the number of uses for each referenced region snapshot. + use db::schema::region_snapshot::dsl as rs_dsl; + for read_only_target in &crucible_targets.read_only_targets + { + diesel::update(rs_dsl::region_snapshot) + .filter( + rs_dsl::snapshot_addr + .eq(read_only_target.clone()), + ) + .filter(rs_dsl::deleting.eq(false)) + .set( + rs_dsl::volume_references + .eq(rs_dsl::volume_references + 1), + ) + .execute_async(&conn) + .await?; + } + + Ok(volume) + } }) .await - .map_err(|e| match e { - TxnError::CustomError(VolumeCreationError::Public(e)) => e, - - _ => { - Error::internal_error(&format!("Transaction error: {}", e)) + .map_err(|e| { + if let Some(err) = err.take() { + match err { + VolumeCreationError::Public(err) => err, + VolumeCreationError::SerdeError(err) => { + Error::internal_error(&format!( + "Transaction error: {}", + err + )) + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) } }) } @@ -192,16 +195,12 @@ impl DataStore { #[derive(Debug, thiserror::Error)] enum VolumeGetError { - #[error("Error during volume_checkout: {0}")] - DieselError(#[from] diesel::result::Error), - #[error("Serde error during volume_checkout: {0}")] SerdeError(#[from] serde_json::Error), #[error("Updated {0} database rows, expected {1}")] UnexpectedDatabaseUpdate(usize, usize), } - type TxnError = TransactionError; // We perform a transaction here, to be sure that on completion // of this, the database contains an updated version of the @@ -209,141 +208,141 @@ impl DataStore { // types that require it). The generation number (along with the // rest of the volume data) that was in the database is what is // returned to the caller. - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - // Grab the volume in question. - let volume = dsl::volume - .filter(dsl::id.eq(volume_id)) - .select(Volume::as_select()) - .get_result_async(&conn) - .await?; - - // Turn the volume.data into the VolumeConstructionRequest - let vcr: VolumeConstructionRequest = - serde_json::from_str(volume.data()).map_err(|e| { - TxnError::CustomError(VolumeGetError::SerdeError(e)) - })?; - - // Look to see if the VCR is a Volume type, and if so, look at - // its sub_volumes. If they are of type Region, then we need - // to update their generation numbers and record that update - // back to the database. We return to the caller whatever the - // original volume data was we pulled from the database. - match vcr { - VolumeConstructionRequest::Volume { - id, - block_size, - sub_volumes, - read_only_parent, - } => { - let mut update_needed = false; - let mut new_sv = Vec::new(); - for sv in sub_volumes { - match sv { - VolumeConstructionRequest::Region { - block_size, - blocks_per_extent, - extent_count, - opts, - gen, - } => { - update_needed = true; - new_sv.push( - VolumeConstructionRequest::Region { - block_size, - blocks_per_extent, - extent_count, - opts, - gen: gen + 1, - }, - ); - } - _ => { - new_sv.push(sv); + let err = OptionalError::new(); + let conn = self.pool_connection_unauthorized().await?; + + self.transaction_retry_wrapper("volume_checkout") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + // Grab the volume in question. + let volume = dsl::volume + .filter(dsl::id.eq(volume_id)) + .select(Volume::as_select()) + .get_result_async(&conn) + .await?; + + // Turn the volume.data into the VolumeConstructionRequest + let vcr: VolumeConstructionRequest = + serde_json::from_str(volume.data()).map_err(|e| { + err.bail(VolumeGetError::SerdeError(e)) + })?; + + // Look to see if the VCR is a Volume type, and if so, look at + // its sub_volumes. If they are of type Region, then we need + // to update their generation numbers and record that update + // back to the database. We return to the caller whatever the + // original volume data was we pulled from the database. + match vcr { + VolumeConstructionRequest::Volume { + id, + block_size, + sub_volumes, + read_only_parent, + } => { + let mut update_needed = false; + let mut new_sv = Vec::new(); + for sv in sub_volumes { + match sv { + VolumeConstructionRequest::Region { + block_size, + blocks_per_extent, + extent_count, + opts, + gen, + } => { + update_needed = true; + new_sv.push( + VolumeConstructionRequest::Region { + block_size, + blocks_per_extent, + extent_count, + opts, + gen: gen + 1, + }, + ); + } + _ => { + new_sv.push(sv); + } } } - } - // Only update the volume data if we found the type - // of volume that needed it. - if update_needed { - // Create a new VCR and fill in the contents - // from what the original volume had, but with our - // updated sub_volume records. - let new_vcr = VolumeConstructionRequest::Volume { - id, - block_size, - sub_volumes: new_sv, - read_only_parent, - }; - - let new_volume_data = serde_json::to_string( - &new_vcr, - ) - .map_err(|e| { - TxnError::CustomError( - VolumeGetError::SerdeError(e), - ) - })?; + // Only update the volume data if we found the type + // of volume that needed it. + if update_needed { + // Create a new VCR and fill in the contents + // from what the original volume had, but with our + // updated sub_volume records. + let new_vcr = VolumeConstructionRequest::Volume { + id, + block_size, + sub_volumes: new_sv, + read_only_parent, + }; - // Update the original volume_id with the new - // volume.data. - use db::schema::volume::dsl as volume_dsl; - let num_updated = - diesel::update(volume_dsl::volume) - .filter(volume_dsl::id.eq(volume_id)) - .set(volume_dsl::data.eq(new_volume_data)) - .execute_async(&conn) - .await?; + let new_volume_data = serde_json::to_string( + &new_vcr, + ) + .map_err(|e| { + err.bail(VolumeGetError::SerdeError(e)) + })?; - // This should update just one row. If it does - // not, then something is terribly wrong in the - // database. - if num_updated != 1 { - return Err(TxnError::CustomError( - VolumeGetError::UnexpectedDatabaseUpdate( - num_updated, - 1, - ), - )); + // Update the original volume_id with the new + // volume.data. + use db::schema::volume::dsl as volume_dsl; + let num_updated = + diesel::update(volume_dsl::volume) + .filter(volume_dsl::id.eq(volume_id)) + .set(volume_dsl::data.eq(new_volume_data)) + .execute_async(&conn) + .await?; + + // This should update just one row. If it does + // not, then something is terribly wrong in the + // database. + if num_updated != 1 { + return Err(err.bail( + VolumeGetError::UnexpectedDatabaseUpdate( + num_updated, + 1, + ), + )); + } } } + VolumeConstructionRequest::Region { + block_size: _, + blocks_per_extent: _, + extent_count: _, + opts: _, + gen: _, + } => { + // We don't support a pure Region VCR at the volume + // level in the database, so this choice should + // never be encountered, but I want to know if it is. + panic!("Region not supported as a top level volume"); + } + VolumeConstructionRequest::File { + id: _, + block_size: _, + path: _, + } + | VolumeConstructionRequest::Url { + id: _, + block_size: _, + url: _, + } => {} } - VolumeConstructionRequest::Region { - block_size: _, - blocks_per_extent: _, - extent_count: _, - opts: _, - gen: _, - } => { - // We don't support a pure Region VCR at the volume - // level in the database, so this choice should - // never be encountered, but I want to know if it is. - panic!("Region not supported as a top level volume"); - } - VolumeConstructionRequest::File { - id: _, - block_size: _, - path: _, - } - | VolumeConstructionRequest::Url { - id: _, - block_size: _, - url: _, - } => {} + Ok(volume) } - Ok(volume) }) .await - .map_err(|e| match e { - TxnError::CustomError(VolumeGetError::DieselError(e)) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - - _ => { - Error::internal_error(&format!("Transaction error: {}", e)) + .map_err(|e| { + if let Some(err) = err.take() { + return Error::internal_error(&format!("Transaction error: {}", err)); } + public_error_from_diesel(e, ErrorHandler::Server) }) } @@ -638,16 +637,12 @@ impl DataStore { ) -> Result { #[derive(Debug, thiserror::Error)] enum RemoveReadOnlyParentError { - #[error("Error removing read only parent: {0}")] - DieselError(#[from] diesel::result::Error), - #[error("Serde error removing read only parent: {0}")] SerdeError(#[from] serde_json::Error), #[error("Updated {0} database rows, expected {1}")] UnexpectedDatabaseUpdate(usize, usize), } - type TxnError = TransactionError; // In this single transaction: // - Get the given volume from the volume_id from the database @@ -663,170 +658,160 @@ impl DataStore { // data from original volume_id. // - Put the new temp VCR into the temp volume.data, update the // temp_volume in the database. - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - // Grab the volume in question. If the volume record was already - // deleted then we can just return. - let volume = { - use db::schema::volume::dsl; - - let volume = dsl::volume - .filter(dsl::id.eq(volume_id)) - .select(Volume::as_select()) - .get_result_async(&conn) - .await - .optional()?; - - let volume = if let Some(v) = volume { - v - } else { - // the volume does not exist, nothing to do. - return Ok(false); + let err = OptionalError::new(); + let conn = self.pool_connection_unauthorized().await?; + self.transaction_retry_wrapper("volume_remove_rop") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + // Grab the volume in question. If the volume record was already + // deleted then we can just return. + let volume = { + use db::schema::volume::dsl; + + let volume = dsl::volume + .filter(dsl::id.eq(volume_id)) + .select(Volume::as_select()) + .get_result_async(&conn) + .await + .optional()?; + + let volume = if let Some(v) = volume { + v + } else { + // the volume does not exist, nothing to do. + return Ok(false); + }; + + if volume.time_deleted.is_some() { + // this volume is deleted, so let whatever is deleting + // it clean it up. + return Ok(false); + } else { + // A volume record exists, and was not deleted, we + // can attempt to remove its read_only_parent. + volume + } }; - if volume.time_deleted.is_some() { - // this volume is deleted, so let whatever is deleting - // it clean it up. - return Ok(false); - } else { - // A volume record exists, and was not deleted, we - // can attempt to remove its read_only_parent. - volume - } - }; - - // If a read_only_parent exists, remove it from volume_id, and - // attach it to temp_volume_id. - let vcr: VolumeConstructionRequest = - serde_json::from_str( - volume.data() - ) - .map_err(|e| { - TxnError::CustomError( - RemoveReadOnlyParentError::SerdeError( - e, - ), + // If a read_only_parent exists, remove it from volume_id, and + // attach it to temp_volume_id. + let vcr: VolumeConstructionRequest = + serde_json::from_str( + volume.data() ) - })?; - - match vcr { - VolumeConstructionRequest::Volume { - id, - block_size, - sub_volumes, - read_only_parent, - } => { - if read_only_parent.is_none() { - // This volume has no read_only_parent - Ok(false) - } else { - // Create a new VCR and fill in the contents - // from what the original volume had. - let new_vcr = VolumeConstructionRequest::Volume { - id, - block_size, - sub_volumes, - read_only_parent: None, - }; - - let new_volume_data = - serde_json::to_string( - &new_vcr + .map_err(|e| { + err.bail( + RemoveReadOnlyParentError::SerdeError( + e, ) - .map_err(|e| { - TxnError::CustomError( - RemoveReadOnlyParentError::SerdeError( - e, - ), - ) - })?; + ) + })?; - // Update the original volume_id with the new - // volume.data. - use db::schema::volume::dsl as volume_dsl; - let num_updated = diesel::update(volume_dsl::volume) - .filter(volume_dsl::id.eq(volume_id)) - .set(volume_dsl::data.eq(new_volume_data)) - .execute_async(&conn) - .await?; - - // This should update just one row. If it does - // not, then something is terribly wrong in the - // database. - if num_updated != 1 { - return Err(TxnError::CustomError( - RemoveReadOnlyParentError::UnexpectedDatabaseUpdate(num_updated, 1), - )); - } + match vcr { + VolumeConstructionRequest::Volume { + id, + block_size, + sub_volumes, + read_only_parent, + } => { + if read_only_parent.is_none() { + // This volume has no read_only_parent + Ok(false) + } else { + // Create a new VCR and fill in the contents + // from what the original volume had. + let new_vcr = VolumeConstructionRequest::Volume { + id, + block_size, + sub_volumes, + read_only_parent: None, + }; - // Make a new VCR, with the information from - // our temp_volume_id, but the read_only_parent - // from the original volume. - let rop_vcr = VolumeConstructionRequest::Volume { - id: temp_volume_id, - block_size, - sub_volumes: vec![], - read_only_parent, - }; - let rop_volume_data = - serde_json::to_string( - &rop_vcr - ) - .map_err(|e| { - TxnError::CustomError( - RemoveReadOnlyParentError::SerdeError( - e, - ), + let new_volume_data = + serde_json::to_string( + &new_vcr ) - })?; - // Update the temp_volume_id with the volume - // data that contains the read_only_parent. - let num_updated = - diesel::update(volume_dsl::volume) - .filter(volume_dsl::id.eq(temp_volume_id)) - .filter(volume_dsl::time_deleted.is_null()) - .set(volume_dsl::data.eq(rop_volume_data)) + .map_err(|e| { + err.bail(RemoveReadOnlyParentError::SerdeError( + e, + )) + })?; + + // Update the original volume_id with the new + // volume.data. + use db::schema::volume::dsl as volume_dsl; + let num_updated = diesel::update(volume_dsl::volume) + .filter(volume_dsl::id.eq(volume_id)) + .set(volume_dsl::data.eq(new_volume_data)) .execute_async(&conn) .await?; - if num_updated != 1 { - return Err(TxnError::CustomError( - RemoveReadOnlyParentError::UnexpectedDatabaseUpdate(num_updated, 1), - )); + + // This should update just one row. If it does + // not, then something is terribly wrong in the + // database. + if num_updated != 1 { + return Err(err.bail(RemoveReadOnlyParentError::UnexpectedDatabaseUpdate(num_updated, 1))); + } + + // Make a new VCR, with the information from + // our temp_volume_id, but the read_only_parent + // from the original volume. + let rop_vcr = VolumeConstructionRequest::Volume { + id: temp_volume_id, + block_size, + sub_volumes: vec![], + read_only_parent, + }; + let rop_volume_data = + serde_json::to_string( + &rop_vcr + ) + .map_err(|e| { + err.bail(RemoveReadOnlyParentError::SerdeError( + e, + )) + })?; + // Update the temp_volume_id with the volume + // data that contains the read_only_parent. + let num_updated = + diesel::update(volume_dsl::volume) + .filter(volume_dsl::id.eq(temp_volume_id)) + .filter(volume_dsl::time_deleted.is_null()) + .set(volume_dsl::data.eq(rop_volume_data)) + .execute_async(&conn) + .await?; + if num_updated != 1 { + return Err(err.bail(RemoveReadOnlyParentError::UnexpectedDatabaseUpdate(num_updated, 1))); + } + Ok(true) } - Ok(true) } - } - VolumeConstructionRequest::File { id: _, block_size: _, path: _ } - | VolumeConstructionRequest::Region { - block_size: _, - blocks_per_extent: _, - extent_count: _, - opts: _, - gen: _ } - | VolumeConstructionRequest::Url { id: _, block_size: _, url: _ } => { - // Volume has a format that does not contain ROPs - Ok(false) + VolumeConstructionRequest::File { id: _, block_size: _, path: _ } + | VolumeConstructionRequest::Region { + block_size: _, + blocks_per_extent: _, + extent_count: _, + opts: _, + gen: _ } + | VolumeConstructionRequest::Url { id: _, block_size: _, url: _ } => { + // Volume has a format that does not contain ROPs + Ok(false) + } } } }) .await - .map_err(|e| match e { - TxnError::CustomError( - RemoveReadOnlyParentError::DieselError(e), - ) => public_error_from_diesel( - e, - ErrorHandler::Server, - ), - - _ => { - Error::internal_error(&format!("Transaction error: {}", e)) + .map_err(|e| { + if let Some(err) = err.take() { + return Error::internal_error(&format!("Transaction error: {}", err)); } + public_error_from_diesel(e, ErrorHandler::Server) }) } } -#[derive(Default, Debug, Serialize, Deserialize)] +#[derive(Default, Clone, Debug, Serialize, Deserialize)] pub struct CrucibleTargets { pub read_only_targets: Vec, } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 6db99465a3..069ce63028 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -12,7 +12,6 @@ use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::fixed_data::vpc::SERVICES_VPC_ID; use crate::db::identity::Resource; use crate::db::model::IncompleteVpc; @@ -37,7 +36,7 @@ use crate::db::queries::vpc::InsertVpcQuery; use crate::db::queries::vpc::VniSearchIter; use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery; use crate::db::queries::vpc_subnet::SubnetError; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; @@ -580,53 +579,65 @@ impl DataStore { .set(dsl::time_deleted.eq(now)); let rules_is_empty = rules.is_empty(); - let insert_new_query = Vpc::insert_resource( - authz_vpc.id(), - diesel::insert_into(dsl::vpc_firewall_rule).values(rules), - ); - #[derive(Debug)] enum FirewallUpdateError { CollectionNotFound, } - type TxnError = TransactionError; + + let err = OptionalError::new(); // TODO-scalability: Ideally this would be a CTE so we don't need to // hold a transaction open across multiple roundtrips from the database, // but for now we're using a transaction due to the severely decreased // legibility of CTEs via diesel right now. - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - delete_old_query.execute_async(&conn).await?; - - // The generation count update on the vpc table row will take a - // write lock on the row, ensuring that the vpc was not deleted - // concurently. - if rules_is_empty { - return Ok(vec![]); - } - insert_new_query + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("vpc_update_firewall_rules") + .transaction(&conn, |conn| { + let err = err.clone(); + let delete_old_query = delete_old_query.clone(); + let rules = rules.clone(); + async move { + delete_old_query.execute_async(&conn).await?; + + // The generation count update on the vpc table row will take a + // write lock on the row, ensuring that the vpc was not deleted + // concurently. + if rules_is_empty { + return Ok(vec![]); + } + Vpc::insert_resource( + authz_vpc.id(), + diesel::insert_into(dsl::vpc_firewall_rule) + .values(rules), + ) .insert_and_get_results_async(&conn) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => { - TxnError::CustomError( - FirewallUpdateError::CollectionNotFound, - ) + err.bail(FirewallUpdateError::CollectionNotFound) } - AsyncInsertError::DatabaseError(e) => e.into(), + AsyncInsertError::DatabaseError(e) => e, }) + } }) .await - .map_err(|e| match e { - TxnError::CustomError( - FirewallUpdateError::CollectionNotFound, - ) => Error::not_found_by_id(ResourceType::Vpc, &authz_vpc.id()), - TxnError::Database(e) => public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_vpc), - ), + .map_err(|e| { + if let Some(err) = err.take() { + match err { + FirewallUpdateError::CollectionNotFound => { + Error::not_found_by_id( + ResourceType::Vpc, + &authz_vpc.id(), + ) + } + } + } else { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_vpc), + ) + } }) } diff --git a/nexus/db-queries/src/db/error.rs b/nexus/db-queries/src/db/error.rs index cbe2b0a71f..fc7f30da93 100644 --- a/nexus/db-queries/src/db/error.rs +++ b/nexus/db-queries/src/db/error.rs @@ -17,7 +17,7 @@ pub enum TransactionError { /// The customizable error type. /// /// This error should be used for all non-Diesel transaction failures. - #[error("Custom transaction error; {0}")] + #[error("Custom transaction error: {0}")] CustomError(T), /// The Diesel error type. @@ -28,31 +28,61 @@ pub enum TransactionError { Database(#[from] DieselError), } +pub fn retryable(error: &DieselError) -> bool { + match error { + DieselError::DatabaseError(kind, boxed_error_information) => match kind + { + DieselErrorKind::SerializationFailure => { + return boxed_error_information + .message() + .starts_with("restart transaction"); + } + _ => false, + }, + _ => false, + } +} + +/// Identifies if the error is retryable or not. +pub enum MaybeRetryable { + /// The error isn't retryable. + NotRetryable(T), + /// The error is retryable. + Retryable(DieselError), +} + +impl TransactionError { + /// Identifies that the error could be returned from a Diesel transaction. + /// + /// Allows callers to propagate arbitrary errors out of transaction contexts + /// without losing information that might be valuable to the calling context, + /// such as "does this particular error indicate that the entire transaction + /// should retry?". + pub fn retryable(self) -> MaybeRetryable { + use MaybeRetryable::*; + + match self { + TransactionError::Database(err) if retryable(&err) => { + Retryable(err) + } + _ => NotRetryable(self), + } + } +} + impl From for TransactionError { fn from(err: PublicError) -> Self { TransactionError::CustomError(err) } } -impl TransactionError { - /// Based on [the CRDB][1] docs, return true if this transaction must be - /// retried. - /// - /// [1]: https://www.cockroachlabs.com/docs/v23.1/transaction-retry-error-reference#client-side-retry-handling - pub fn retry_transaction(&self) -> bool { - match &self { - Self::Database(DieselError::DatabaseError( - kind, - boxed_error_information, - )) => match kind { - DieselErrorKind::SerializationFailure => { - return boxed_error_information - .message() - .starts_with("restart transaction"); - } - _ => false, - }, - _ => false, +impl From> for PublicError { + fn from(err: TransactionError) -> Self { + match err { + TransactionError::CustomError(err) => err, + TransactionError::Database(err) => { + public_error_from_diesel(err, ErrorHandler::Server) + } } } } diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index 72a32f562c..028694dc4b 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -231,6 +231,11 @@ impl<'a> LookupPath<'a> { RouterRoute::PrimaryKey(Root { lookup_root: self }, id) } + /// Select a resource of type FloatingIp, identified by its id + pub fn floating_ip_id(self, id: Uuid) -> FloatingIp<'a> { + FloatingIp::PrimaryKey(Root { lookup_root: self }, id) + } + // Fleet-level resources /// Select a resource of type ConsoleSession, identified by its `token` @@ -632,7 +637,7 @@ lookup_resource! { lookup_resource! { name = "Project", ancestors = [ "Silo" ], - children = [ "Disk", "Instance", "Vpc", "Snapshot", "ProjectImage" ], + children = [ "Disk", "Instance", "Vpc", "Snapshot", "ProjectImage", "FloatingIp" ], lookup_by_name = true, soft_deletes = true, primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] @@ -728,6 +733,15 @@ lookup_resource! { primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] } +lookup_resource! { + name = "FloatingIp", + ancestors = [ "Silo", "Project" ], + children = [], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + // Miscellaneous resources nested directly below "Fleet" lookup_resource! { diff --git a/nexus/db-queries/src/db/mod.rs b/nexus/db-queries/src/db/mod.rs index b7c7079b54..924eab363f 100644 --- a/nexus/db-queries/src/db/mod.rs +++ b/nexus/db-queries/src/db/mod.rs @@ -17,11 +17,12 @@ mod config; mod cte_utils; // This is marked public for use by the integration tests pub mod datastore; -mod error; +pub(crate) mod error; mod explain; pub mod fixed_data; pub mod lookup; -mod pagination; +// Public for doctests. +pub mod pagination; mod pool; // This is marked public because the error types are used elsewhere, e.g., in // sagas. @@ -42,7 +43,7 @@ pub use nexus_db_model::schema; pub use crate::db::error::TransactionError; pub use config::Config; pub use datastore::DataStore; -pub use pool::Pool; +pub use pool::{DbConnection, Pool}; pub use saga_recovery::{recover, CompletionTask, RecoveryTask}; pub use saga_types::SecId; pub use sec_store::CockroachDbSecStore; diff --git a/nexus/db-queries/src/db/pagination.rs b/nexus/db-queries/src/db/pagination.rs index dd7daab14f..4fc1cf5966 100644 --- a/nexus/db-queries/src/db/pagination.rs +++ b/nexus/db-queries/src/db/pagination.rs @@ -16,6 +16,7 @@ use diesel::AppearsOnTable; use diesel::Column; use diesel::{ExpressionMethods, QueryDsl}; use omicron_common::api::external::DataPageParams; +use std::num::NonZeroU32; // Shorthand alias for "the SQL type of the whole table". type TableSqlType = ::SqlType; @@ -169,6 +170,145 @@ where } } +/// Helper for querying a large number of records from the database in batches +/// +/// Without this helper: a typical way to perform paginated queries would be to +/// invoke some existing "list" function in the datastore that itself is +/// paginated. Such functions accept a `pagparams: &DataPageParams` argument +/// that uses a marker to identify where the next page of results starts. For +/// the first call, the marker inside `pagparams` is `None`. For subsequent +/// calls, it's typically some field from the last item returned in the previous +/// page. You're finished when you get a result set smaller than the batch +/// size. +/// +/// This helper takes care of most of the logic for you. To use this, you first +/// create a `Paginator` with a specific batch_size. Then you call `next()` in +/// a loop. Each iteration will provide you with a `DataPageParams` to use to +/// call your list function. When you've fetched the next page, you have to +/// let the helper look at it to determine if there's another page to fetch and +/// what marker to use. +/// +/// ## Example +/// +/// ``` +/// use nexus_db_queries::db::pagination::Paginator; +/// use omicron_common::api::external::DataPageParams; +/// +/// let batch_size = std::num::NonZeroU32::new(3).unwrap(); +/// +/// // Assume you've got an existing paginated "list items" function. +/// // This simple implementation returns a few full batches, then a partial +/// // batch. +/// type Marker = u32; +/// type Item = u32; +/// let do_query = |pagparams: &DataPageParams<'_, Marker> | { +/// match pagparams.marker { +/// None => (0..batch_size.get()).collect(), +/// Some(x) if *x < 2 * batch_size.get() => (x+1..x+1+batch_size.get()).collect(), +/// Some(x) => vec![*x + 1], +/// } +/// }; +/// +/// // This closure translates from one of the returned item to the field in +/// // that item that servers as the marker. This example is contrived. +/// let item2marker: &dyn Fn(&Item) -> Marker = &|u: &u32| *u; +/// +/// let mut all_records = Vec::new(); +/// let mut paginator = Paginator::new(batch_size); +/// while let Some(p) = paginator.next() { +/// let records_batch = do_query(&p.current_pagparams()); +/// paginator = p.found_batch(&records_batch, item2marker); +/// all_records.extend(records_batch.into_iter()); +/// } +/// +/// // Results are in `all_records`. +/// assert_eq!(all_records, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); +/// ``` +/// +/// ## Design notes +/// +/// The separation of `Paginator` and `PaginatorHelper` is aimed at making it +/// harder to misuse this interface. We could skip the helper altogether and +/// just have `Paginator::next()` return the DatePageParams directly. But you'd +/// still need a `Paginator::found_batch()`. And it would be easy to forget to +/// call this, leading to an infinite loop at runtime. To avoid this mistake, +/// `Paginator::next()` consumes `self`. You can't get another `Paginator` back +/// until you use `PaginatorHelper::found_batch()`. That also consumes `self` +/// so that you can't keep using the old `DataPageParams`. +pub struct Paginator { + batch_size: NonZeroU32, + state: PaginatorState, +} + +impl Paginator { + pub fn new(batch_size: NonZeroU32) -> Paginator { + Paginator { batch_size, state: PaginatorState::Initial } + } + + pub fn next(self) -> Option> { + match self.state { + PaginatorState::Initial => Some(PaginatorHelper { + batch_size: self.batch_size, + marker: None, + }), + PaginatorState::Middle { marker } => Some(PaginatorHelper { + batch_size: self.batch_size, + marker: Some(marker), + }), + PaginatorState::Done => None, + } + } +} + +enum PaginatorState { + Initial, + Middle { marker: N }, + Done, +} + +pub struct PaginatorHelper { + batch_size: NonZeroU32, + marker: Option, +} + +impl PaginatorHelper { + /// Returns the `DatePageParams` to use to fetch the next page of results + pub fn current_pagparams(&self) -> DataPageParams<'_, N> { + DataPageParams { + marker: self.marker.as_ref(), + direction: dropshot::PaginationOrder::Ascending, + limit: self.batch_size, + } + } + + /// Report a page of results + /// + /// This function looks at the returned results to determine whether we've + /// finished iteration or whether we need to fetch another page (and if so, + /// this determines the marker for the next fetch operation). + /// + /// This function returns a `Paginator` used to make the next request. See + /// the example on `Paginator` for usage. + pub fn found_batch( + self, + batch: &[T], + item2marker: &dyn Fn(&T) -> N, + ) -> Paginator { + let state = + if batch.len() < usize::try_from(self.batch_size.get()).unwrap() { + PaginatorState::Done + } else { + // self.batch_size is non-zero, so if we got at least that many + // items, then there's at least one. + let last = batch.iter().last().unwrap(); + let marker = item2marker(last); + PaginatorState::Middle { marker } + }; + + Paginator { batch_size: self.batch_size, state } + } +} + #[cfg(test)] mod test { use super::*; @@ -433,4 +573,48 @@ mod test { let _ = db.cleanup().await; logctx.cleanup_successful(); } + + #[test] + fn test_paginator() { + // The doctest exercises a basic case for Paginator. Here we test some + // edge cases. + let batch_size = std::num::NonZeroU32::new(3).unwrap(); + + type Marker = u32; + #[derive(Debug, PartialEq, Eq)] + struct Item { + value: String, + marker: Marker, + } + + let do_list = + |query: &dyn Fn(&DataPageParams<'_, Marker>) -> Vec| { + let mut all_records = Vec::new(); + let mut paginator = Paginator::new(batch_size); + while let Some(p) = paginator.next() { + let records_batch = query(&p.current_pagparams()); + paginator = + p.found_batch(&records_batch, &|i: &Item| i.marker); + all_records.extend(records_batch.into_iter()); + } + all_records + }; + + fn mkitem(v: u32) -> Item { + Item { value: v.to_string(), marker: v } + } + + // Trivial case: first page is empty + assert_eq!(Vec::::new(), do_list(&|_| Vec::new())); + + // Exactly one batch-size worth of items + // (exercises the cases where the last non-empty batch is full, and + // where any batch is empty) + let my_query = + |pagparams: &DataPageParams<'_, Marker>| match &pagparams.marker { + None => (0..batch_size.get()).map(mkitem).collect(), + Some(_) => Vec::new(), + }; + assert_eq!(vec![mkitem(0), mkitem(1), mkitem(2)], do_list(&my_query)); + } } diff --git a/nexus/db-queries/src/db/queries/external_ip.rs b/nexus/db-queries/src/db/queries/external_ip.rs index cf182e080d..4e5f59e79c 100644 --- a/nexus/db-queries/src/db/queries/external_ip.rs +++ b/nexus/db-queries/src/db/queries/external_ip.rs @@ -98,7 +98,8 @@ const MAX_PORT: u16 = u16::MAX; /// AS kind, /// candidate_ip AS ip, /// CAST(candidate_first_port AS INT4) AS first_port, -/// CAST(candidate_last_port AS INT4) AS last_port +/// CAST(candidate_last_port AS INT4) AS last_port, +/// AS project_id /// FROM /// SELECT * FROM ( /// -- Select all IP addresses by pool and range. @@ -371,6 +372,13 @@ impl NextExternalIp { out.push_identifier(dsl::first_port::NAME)?; out.push_sql(", CAST(candidate_last_port AS INT4) AS "); out.push_identifier(dsl::last_port::NAME)?; + out.push_sql(", "); + + // Project ID, possibly null + out.push_bind_param::, Option>(self.ip.project_id())?; + out.push_sql(" AS "); + out.push_identifier(dsl::project_id::NAME)?; + out.push_sql(" FROM ("); self.push_address_sequence_subquery(out.reborrow())?; out.push_sql(") CROSS JOIN ("); diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs index 84a81a7b7a..1dbe57da6f 100644 --- a/nexus/db-queries/src/db/queries/network_interface.rs +++ b/nexus/db-queries/src/db/queries/network_interface.rs @@ -5,6 +5,7 @@ //! Queries for inserting and deleting network interfaces. use crate::db; +use crate::db::error::{public_error_from_diesel, retryable, ErrorHandler}; use crate::db::model::IncompleteNetworkInterface; use crate::db::pool::DbConnection; use crate::db::queries::next_item::DefaultShiftGenerator; @@ -120,6 +121,8 @@ pub enum InsertError { InstanceMustBeStopped(Uuid), /// The instance does not exist at all, or is in the destroyed state. InstanceNotFound(Uuid), + /// The operation occurred within a transaction, and is retryable + Retryable(DieselError), /// Any other error External(external::Error), } @@ -135,7 +138,6 @@ impl InsertError { e: DieselError, interface: &IncompleteNetworkInterface, ) -> Self { - use crate::db::error; match e { // Catch the specific errors designed to communicate the failures we // want to distinguish @@ -143,9 +145,9 @@ impl InsertError { decode_database_error(e, interface) } // Any other error at all is a bug - _ => InsertError::External(error::public_error_from_diesel( + _ => InsertError::External(public_error_from_diesel( e, - error::ErrorHandler::Server, + ErrorHandler::Server, )), } } @@ -209,6 +211,9 @@ impl InsertError { InsertError::InstanceNotFound(id) => { external::Error::not_found_by_id(external::ResourceType::Instance, &id) } + InsertError::Retryable(err) => { + public_error_from_diesel(err, ErrorHandler::Server) + } InsertError::External(e) => e, } } @@ -290,6 +295,10 @@ fn decode_database_error( r#"uuid: incorrect UUID length: non-unique-subnets"#, ); + if retryable(&err) { + return InsertError::Retryable(err); + } + match err { // If the address allocation subquery fails, we'll attempt to insert // NULL for the `ip` column. This checks that the non-NULL constraint on diff --git a/nexus/db-queries/src/lib.rs b/nexus/db-queries/src/lib.rs index a693f7ff42..5d1927ebc7 100644 --- a/nexus/db-queries/src/lib.rs +++ b/nexus/db-queries/src/lib.rs @@ -9,6 +9,7 @@ pub mod authz; pub mod context; pub mod db; pub mod provisioning; +pub mod transaction_retry; #[macro_use] extern crate slog; diff --git a/nexus/db-queries/src/transaction_retry.rs b/nexus/db-queries/src/transaction_retry.rs new file mode 100644 index 0000000000..c474b729f8 --- /dev/null +++ b/nexus/db-queries/src/transaction_retry.rs @@ -0,0 +1,341 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Helper types for performing automatic transaction retries + +use async_bb8_diesel::AsyncConnection; +use chrono::Utc; +use diesel::result::Error as DieselError; +use oximeter::{types::Sample, Metric, MetricsError, Target}; +use rand::{thread_rng, Rng}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +// Identifies "which" transaction is retrying +#[derive(Debug, Clone, Target)] +struct DatabaseTransaction { + name: String, +} + +// Identifies that a retry has occurred, and track how long +// the transaction took (either since starting, or since the last +// retry failure was recorded). +#[derive(Debug, Clone, Metric)] +struct RetryData { + #[datum] + latency: f64, + attempt: u32, +} + +// Collects all transaction retry samples +#[derive(Debug, Default, Clone)] +pub(crate) struct Producer { + samples: Arc>>, +} + +impl Producer { + pub(crate) fn new() -> Self { + Self { samples: Arc::new(Mutex::new(vec![])) } + } + + fn append( + &self, + transaction: &DatabaseTransaction, + data: &RetryData, + ) -> Result<(), MetricsError> { + let sample = Sample::new_with_timestamp(Utc::now(), transaction, data)?; + self.samples.lock().unwrap().push(sample); + Ok(()) + } +} + +struct RetryHelperInner { + start: chrono::DateTime, + attempts: u32, +} + +impl RetryHelperInner { + fn new() -> Self { + Self { start: Utc::now(), attempts: 1 } + } + + fn tick(&mut self) -> Self { + let start = self.start; + let attempts = self.attempts; + + self.start = Utc::now(); + self.attempts += 1; + + Self { start, attempts } + } +} + +/// Helper utility for tracking retry attempts and latency. +/// Intended to be used from within "transaction_async_with_retry". +pub struct RetryHelper { + producer: Producer, + name: &'static str, + inner: Mutex, +} + +const MIN_RETRY_BACKOFF: Duration = Duration::from_millis(0); +const MAX_RETRY_BACKOFF: Duration = Duration::from_millis(50); +const MAX_RETRY_ATTEMPTS: u32 = 10; + +impl RetryHelper { + /// Creates a new RetryHelper, and starts a timer tracking the transaction + /// duration. + pub(crate) fn new(producer: &Producer, name: &'static str) -> Self { + Self { + producer: producer.clone(), + name, + inner: Mutex::new(RetryHelperInner::new()), + } + } + + /// Calls the function "f" in an asynchronous, retryable transaction. + pub async fn transaction( + self, + conn: &async_bb8_diesel::Connection, + f: Func, + ) -> Result + where + R: Send + 'static, + Fut: std::future::Future> + Send, + Func: Fn(async_bb8_diesel::Connection) -> Fut + + Send + + Sync, + { + conn.transaction_async_with_retry(f, self.as_callback()).await + } + + // Called upon retryable transaction failure. + // + // This function: + // - Appends a metric identifying the duration of the transaction operation + // - Performs a random (uniform) backoff (limited to less than 50 ms) + // - Returns "true" if the transaction should be restarted + async fn retry_callback(&self) -> bool { + // Look at the current attempt and start time so we can log this + // information before we start sleeping. + let (start, attempt) = { + let inner = self.inner.lock().unwrap(); + (inner.start, inner.attempts) + }; + + let latency = (Utc::now() - start) + .to_std() + .unwrap_or(Duration::ZERO) + .as_secs_f64(); + + let _ = self.producer.append( + &DatabaseTransaction { name: self.name.into() }, + &RetryData { latency, attempt }, + ); + + // This backoff is not exponential, but I'm not sure we actually want + // that much backoff here. If we're repeatedly failing, it would + // probably be better to fail the operation, at which point Oximeter + // will keep track of the failing transaction and identify that it's a + // high-priority target for CTE conversion. + let duration = { + let mut rng = thread_rng(); + rng.gen_range(MIN_RETRY_BACKOFF..MAX_RETRY_BACKOFF) + }; + tokio::time::sleep(duration).await; + + // Now that we've finished sleeping, reset the timer and bump the number + // of attempts we've tried. + let inner = self.inner.lock().unwrap().tick(); + return inner.attempts < MAX_RETRY_ATTEMPTS; + } + + /// Converts this function to a retryable callback that can be used from + /// "transaction_async_with_retry". + pub(crate) fn as_callback( + self, + ) -> impl Fn() -> futures::future::BoxFuture<'static, bool> { + let r = Arc::new(self); + move || { + let r = r.clone(); + Box::pin(async move { r.retry_callback().await }) + } + } +} + +impl oximeter::Producer for Producer { + fn produce( + &mut self, + ) -> Result + 'static>, MetricsError> { + let samples = std::mem::take(&mut *self.samples.lock().unwrap()); + Ok(Box::new(samples.into_iter())) + } +} + +/// Helper utility for passing non-retryable errors out-of-band from +/// transactions. +/// +/// Transactions prefer to act on the `diesel::result::Error` type, +/// but transaction users may want more meaningful error types. +/// This utility helps callers safely propagate back Diesel errors while +/// retaining auxiliary error info. +pub struct OptionalError(Arc>>); + +impl Clone for OptionalError { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl OptionalError { + pub fn new() -> Self { + Self(Arc::new(Mutex::new(None))) + } + + /// Sets "Self" to the value of `error` and returns `DieselError::RollbackTransaction`. + pub fn bail(&self, err: E) -> DieselError { + (*self.0.lock().unwrap()).replace(err); + DieselError::RollbackTransaction + } + + /// If `diesel_error` is retryable, returns it without setting Self. + /// + /// Otherwise, sets "Self" to the value of `err`, and returns + /// `DieselError::RollbackTransaction`. + pub fn bail_retryable_or( + &self, + diesel_error: DieselError, + err: E, + ) -> DieselError { + self.bail_retryable_or_else(diesel_error, |_diesel_error| err) + } + + /// If `diesel_error` is retryable, returns it without setting Self. + /// + /// Otherwise, sets "Self" to the value of `f` applied to `diesel_err`, and + /// returns `DieselError::RollbackTransaction`. + pub fn bail_retryable_or_else( + &self, + diesel_error: DieselError, + f: F, + ) -> DieselError + where + F: FnOnce(DieselError) -> E, + { + if crate::db::error::retryable(&diesel_error) { + return diesel_error; + } else { + self.bail(f(diesel_error)) + } + } + + /// If "Self" was previously set to a non-retryable error, return it. + pub fn take(self) -> Option { + (*self.0.lock().unwrap()).take() + } +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::db::datastore::datastore_test; + use nexus_test_utils::db::test_setup_database; + use omicron_test_utils::dev; + use oximeter::types::FieldValue; + + // If a transaction is explicitly rolled back, we should not expect any + // samples to be taken. With no retries, this is just a normal operation + // failure. + #[tokio::test] + async fn test_transaction_rollback_produces_no_samples() { + let logctx = dev::test_setup_log( + "test_transaction_rollback_produces_no_samples", + ); + let mut db = test_setup_database(&logctx.log).await; + let (_opctx, datastore) = datastore_test(&logctx, &db).await; + + let conn = datastore.pool_connection_for_tests().await.unwrap(); + + datastore + .transaction_retry_wrapper( + "test_transaction_rollback_produces_no_samples", + ) + .transaction(&conn, |_conn| async move { + Err::<(), _>(diesel::result::Error::RollbackTransaction) + }) + .await + .expect_err("Should have failed"); + + let samples = datastore + .transaction_retry_producer() + .samples + .lock() + .unwrap() + .clone(); + assert_eq!(samples, vec![]); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + + // If a transaction fails with a retryable error, we record samples, + // providing oximeter-level information about the attempts. + #[tokio::test] + async fn test_transaction_retry_produces_samples() { + let logctx = + dev::test_setup_log("test_transaction_retry_produces_samples"); + let mut db = test_setup_database(&logctx.log).await; + let (_opctx, datastore) = datastore_test(&logctx, &db).await; + + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore + .transaction_retry_wrapper( + "test_transaction_retry_produces_samples", + ) + .transaction(&conn, |_conn| async move { + Err::<(), _>(diesel::result::Error::DatabaseError( + diesel::result::DatabaseErrorKind::SerializationFailure, + Box::new("restart transaction: Retry forever!".to_string()), + )) + }) + .await + .expect_err("Should have failed"); + + let samples = datastore + .transaction_retry_producer() + .samples + .lock() + .unwrap() + .clone(); + assert_eq!(samples.len(), MAX_RETRY_ATTEMPTS as usize); + + for i in 0..samples.len() { + let sample = &samples[i]; + + assert_eq!( + sample.timeseries_name, + "database_transaction:retry_data" + ); + + let target_fields = sample.sorted_target_fields(); + assert_eq!( + target_fields["name"].value, + FieldValue::String( + "test_transaction_retry_produces_samples".to_string() + ) + ); + + // Attempts are one-indexed + let metric_fields = sample.sorted_metric_fields(); + assert_eq!( + metric_fields["attempt"].value, + FieldValue::U32(u32::try_from(i).unwrap() + 1), + ); + } + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/tests/output/authz-roles.out b/nexus/db-queries/tests/output/authz-roles.out index 963f00f7e8..54fb6481a9 100644 --- a/nexus/db-queries/tests/output/authz-roles.out +++ b/nexus/db-queries/tests/output/authz-roles.out @@ -376,6 +376,20 @@ resource: ProjectImage "silo1-proj1-image1" silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: FloatingIp "silo1-proj1-fip1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Project "silo1-proj2" USER Q R LC RP M MP CC D @@ -488,6 +502,20 @@ resource: ProjectImage "silo1-proj2-image1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: FloatingIp "silo1-proj2-fip1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Silo "silo2" USER Q R LC RP M MP CC D @@ -768,6 +796,20 @@ resource: ProjectImage "silo2-proj1-image1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: FloatingIp "silo2-proj1-fip1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Rack id "c037e882-8b6d-c8b5-bef4-97e848eb0a50" USER Q R LC RP M MP CC D diff --git a/nexus/src/app/background/dns_config.rs b/nexus/src/app/background/dns_config.rs index 654e9c0bf1..805ae813fe 100644 --- a/nexus/src/app/background/dns_config.rs +++ b/nexus/src/app/background/dns_config.rs @@ -166,7 +166,6 @@ mod test { use crate::app::background::init::test::read_internal_dns_zone_id; use crate::app::background::init::test::write_test_dns_generation; use assert_matches::assert_matches; - use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use async_bb8_diesel::AsyncSimpleConnection; use diesel::ExpressionMethods; @@ -237,11 +236,11 @@ mod test { ); // Similarly, wipe all of the state and verify that we handle that okay. + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper("dns_config_test_basic") + .transaction(&conn, |conn| async move { conn.batch_execute_async( nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL, ) @@ -265,7 +264,7 @@ mod test { .execute_async(&conn) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>(()) + Ok(()) }) .await .unwrap(); diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index cfa023a013..d30d2162c4 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -247,14 +247,12 @@ fn init_dns( #[cfg(test)] pub mod test { - use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use dropshot::HandlerTaskMode; use nexus_db_model::DnsGroup; use nexus_db_model::Generation; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; - use nexus_db_queries::db::TransactionError; use nexus_test_utils_macros::nexus_test; use nexus_types::internal_api::params as nexus_params; use nexus_types::internal_api::params::ServiceKind; @@ -446,11 +444,11 @@ pub mod test { datastore: &DataStore, internal_dns_zone_id: Uuid, ) { - type TxnError = TransactionError<()>; { let conn = datastore.pool_connection_for_tests().await.unwrap(); - let _: Result<(), TxnError> = conn - .transaction_async(|conn| async move { + let _: Result<(), _> = datastore + .transaction_retry_wrapper("write_test_dns_generation") + .transaction(&conn, |conn| async move { { use nexus_db_queries::db::model::DnsVersion; use nexus_db_queries::db::schema::dns_version::dsl; diff --git a/nexus/src/app/external_ip.rs b/nexus/src/app/external_ip.rs index 2354e97085..404f597288 100644 --- a/nexus/src/app/external_ip.rs +++ b/nexus/src/app/external_ip.rs @@ -5,11 +5,20 @@ //! External IP addresses for instances use crate::external_api::views::ExternalIp; +use crate::external_api::views::FloatingIp; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::lookup; +use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::IpKind; +use nexus_types::external_api::params; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::LookupResult; +use omicron_common::api::external::NameOrId; impl super::Nexus { pub(crate) async fn instance_list_external_ips( @@ -33,4 +42,82 @@ impl super::Nexus { }) .collect::>()) } + + pub(crate) fn floating_ip_lookup<'a>( + &'a self, + opctx: &'a OpContext, + fip_selector: params::FloatingIpSelector, + ) -> LookupResult> { + match fip_selector { + params::FloatingIpSelector { floating_ip: NameOrId::Id(id), project: None } => { + let floating_ip = + LookupPath::new(opctx, &self.db_datastore).floating_ip_id(id); + Ok(floating_ip) + } + params::FloatingIpSelector { + floating_ip: NameOrId::Name(name), + project: Some(project), + } => { + let floating_ip = self + .project_lookup(opctx, params::ProjectSelector { project })? + .floating_ip_name_owned(name.into()); + Ok(floating_ip) + } + params::FloatingIpSelector { + floating_ip: NameOrId::Id(_), + .. + } => Err(Error::invalid_request( + "when providing Floating IP as an ID project should not be specified", + )), + _ => Err(Error::invalid_request( + "Floating IP should either be UUID or project should be specified", + )), + } + } + + pub(crate) async fn floating_ips_list( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::ListChildren).await?; + + Ok(self + .db_datastore + .floating_ips_list(opctx, &authz_project, pagparams) + .await? + .into_iter() + .map(Into::into) + .collect()) + } + + pub(crate) async fn floating_ip_create( + &self, + opctx: &OpContext, + project_lookup: &lookup::Project<'_>, + params: params::FloatingIpCreate, + ) -> CreateResult { + let (.., authz_project) = + project_lookup.lookup_for(authz::Action::CreateChild).await?; + + Ok(self + .db_datastore + .allocate_floating_ip(opctx, authz_project.id(), params) + .await? + .try_into() + .unwrap()) + } + + pub(crate) async fn floating_ip_delete( + &self, + opctx: &OpContext, + ip_lookup: lookup::FloatingIp<'_>, + ) -> DeleteResult { + let (.., authz_fip, db_fip) = + ip_lookup.fetch_for(authz::Action::Delete).await?; + + self.db_datastore.floating_ip_delete(opctx, &authz_fip, &db_fip).await + } } diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 923bb1777e..0edb2c5ea7 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -5,6 +5,7 @@ //! Virtual Machine Instances use super::MAX_DISKS_PER_INSTANCE; +use super::MAX_EPHEMERAL_IPS_PER_INSTANCE; use super::MAX_EXTERNAL_IPS_PER_INSTANCE; use super::MAX_MEMORY_BYTES_PER_INSTANCE; use super::MAX_NICS_PER_INSTANCE; @@ -52,6 +53,7 @@ use sled_agent_client::types::InstanceProperties; use sled_agent_client::types::InstancePutMigrationIdsBody; use sled_agent_client::types::InstancePutStateBody; use sled_agent_client::types::SourceNatConfig; +use std::matches; use std::net::SocketAddr; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; @@ -168,6 +170,18 @@ impl super::Nexus { MAX_EXTERNAL_IPS_PER_INSTANCE, ))); } + if params + .external_ips + .iter() + .filter(|v| matches!(v, params::ExternalIpCreate::Ephemeral { .. })) + .count() + > MAX_EPHEMERAL_IPS_PER_INSTANCE + { + return Err(Error::invalid_request(&format!( + "An instance may not have more than {} ephemeral IP address", + MAX_EPHEMERAL_IPS_PER_INSTANCE, + ))); + } if let params::InstanceNetworkInterfaceAttachment::Create(ref ifaces) = params.network_interfaces { @@ -885,8 +899,6 @@ impl super::Nexus { .await?; // Collect the external IPs for the instance. - // TODO-correctness: Handle Floating IPs, see - // https://github.com/oxidecomputer/omicron/issues/1334 let (snat_ip, external_ips): (Vec<_>, Vec<_>) = self .db_datastore .instance_lookup_external_ips(&opctx, authz_instance.id()) @@ -895,8 +907,6 @@ impl super::Nexus { .partition(|ip| ip.kind == IpKind::SNat); // Sanity checks on the number and kind of each IP address. - // TODO-correctness: Handle multiple IP addresses, see - // https://github.com/oxidecomputer/omicron/issues/1467 if external_ips.len() > MAX_EXTERNAL_IPS_PER_INSTANCE { return Err(Error::internal_error( format!( @@ -908,8 +918,28 @@ impl super::Nexus { .as_str(), )); } - let external_ips = - external_ips.into_iter().map(|model| model.ip.ip()).collect(); + + // Partition remaining external IPs by class: we can have at most + // one ephemeral ip. + let (ephemeral_ips, floating_ips): (Vec<_>, Vec<_>) = external_ips + .into_iter() + .partition(|ip| ip.kind == IpKind::Ephemeral); + + if ephemeral_ips.len() > MAX_EPHEMERAL_IPS_PER_INSTANCE { + return Err(Error::internal_error( + format!( + "Expected at most {} ephemeral IP for an instance, found {}", + MAX_EPHEMERAL_IPS_PER_INSTANCE, + ephemeral_ips.len() + ) + .as_str(), + )); + } + + let ephemeral_ip = ephemeral_ips.get(0).map(|model| model.ip.ip()); + + let floating_ips = + floating_ips.into_iter().map(|model| model.ip.ip()).collect(); if snat_ip.len() != 1 { return Err(Error::internal_error( "Expected exactly one SNAT IP address for an instance", @@ -985,7 +1015,8 @@ impl super::Nexus { }, nics, source_nat, - external_ips, + ephemeral_ip, + floating_ips, firewall_rules, dhcp_config: sled_agent_client::types::DhcpConfig { dns_servers: self.external_dns_servers.clone(), diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 18c9dae841..d4c2d596f8 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -79,8 +79,13 @@ pub(crate) use nexus_db_queries::db::queries::disk::MAX_DISKS_PER_INSTANCE; pub(crate) const MAX_NICS_PER_INSTANCE: usize = 8; -// TODO-completeness: Support multiple external IPs -pub(crate) const MAX_EXTERNAL_IPS_PER_INSTANCE: usize = 1; +// XXX: Might want to recast as max *floating* IPs, we have at most one +// ephemeral (so bounded in saga by design). +// The value here is arbitrary, but we need *a* limit for the instance +// create saga to have a bounded DAG. We might want to only enforce +// this during instance create (rather than live attach) in future. +pub(crate) const MAX_EXTERNAL_IPS_PER_INSTANCE: usize = 32; +pub(crate) const MAX_EPHEMERAL_IPS_PER_INSTANCE: usize = 1; pub const MAX_VCPU_PER_INSTANCE: u16 = 64; diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 95283faa1c..3804841feb 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -869,7 +869,17 @@ impl super::Nexus { ) -> ListResultVec { debug!(self.log, "Getting latest collection"); // Grab the SPs from the last collection - let limit = NonZeroU32::new(50).unwrap(); + // + // We set a limit of 200 here to give us some breathing room when + // querying for cabooses and RoT pages, each of which is "4 per SP/RoT", + // which in a single fully populated rack works out to (32 sleds + 2 + // switches + 1 psc) * 4 = 140. + // + // This feels bad and probably needs more thought; see + // https://github.com/oxidecomputer/omicron/issues/4621 where this limit + // being too low bit us, and it will link to a more general followup + // issue. + let limit = NonZeroU32::new(200).unwrap(); let collection = self .db_datastore .inventory_get_latest_collection(opctx, limit) diff --git a/nexus/src/app/sagas/disk_create.rs b/nexus/src/app/sagas/disk_create.rs index fe403a7d41..4883afaddc 100644 --- a/nexus/src/app/sagas/disk_create.rs +++ b/nexus/src/app/sagas/disk_create.rs @@ -830,9 +830,7 @@ pub(crate) mod test { app::saga::create_saga_dag, app::sagas::disk_create::Params, app::sagas::disk_create::SagaDiskCreate, external_api::params, }; - use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use diesel::{ ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, }; @@ -972,27 +970,25 @@ pub(crate) mod test { use nexus_db_queries::db::model::VirtualProvisioningCollection; use nexus_db_queries::db::schema::virtual_provisioning_collection::dsl; + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper( + "no_virtual_provisioning_collection_records_using_storage", + ) + .transaction(&conn, |conn| async move { conn.batch_execute_async( nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL, ) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( - dsl::virtual_provisioning_collection - .filter(dsl::virtual_disk_bytes_provisioned.ne(0)) - .select(VirtualProvisioningCollection::as_select()) - .get_results_async::( - &conn, - ) - .await - .unwrap() - .is_empty(), - ) + Ok(dsl::virtual_provisioning_collection + .filter(dsl::virtual_disk_bytes_provisioned.ne(0)) + .select(VirtualProvisioningCollection::as_select()) + .get_results_async::(&conn) + .await + .unwrap() + .is_empty()) }) .await .unwrap() diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index 153e0323e7..5149825842 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -598,35 +598,55 @@ async fn sic_allocate_instance_snat_ip_undo( async fn sic_allocate_instance_external_ip( sagactx: NexusActionContext, ) -> Result<(), ActionError> { + // XXX: may wish to restructure partially: we have at most one ephemeral + // and then at most $n$ floating. let osagactx = sagactx.user_data(); let datastore = osagactx.datastore(); let repeat_saga_params = sagactx.saga_params::()?; let saga_params = repeat_saga_params.saga_params; let ip_index = repeat_saga_params.which; - let ip_params = saga_params.create_params.external_ips.get(ip_index); - let ip_params = match ip_params { - None => { - return Ok(()); - } - Some(ref prs) => prs, + let Some(ip_params) = saga_params.create_params.external_ips.get(ip_index) + else { + return Ok(()); }; let opctx = crate::context::op_context_for_saga_action( &sagactx, &saga_params.serialized_authn, ); let instance_id = repeat_saga_params.instance_id; - let ip_id = repeat_saga_params.new_id; - // Collect the possible pool name for this IP address - let pool_name = match ip_params { + match ip_params { + // Allocate a new IP address from the target, possibly default, pool params::ExternalIpCreate::Ephemeral { ref pool_name } => { - pool_name.as_ref().map(|name| db::model::Name(name.clone())) + let pool_name = + pool_name.as_ref().map(|name| db::model::Name(name.clone())); + let ip_id = repeat_saga_params.new_id; + datastore + .allocate_instance_ephemeral_ip( + &opctx, + ip_id, + instance_id, + pool_name, + ) + .await + .map_err(ActionError::action_failed)?; } - }; - datastore - .allocate_instance_ephemeral_ip(&opctx, ip_id, instance_id, pool_name) - .await - .map_err(ActionError::action_failed)?; + // Set the parent of an existing floating IP to the new instance's ID. + params::ExternalIpCreate::Floating { ref floating_ip_name } => { + let floating_ip_name = db::model::Name(floating_ip_name.clone()); + let (.., authz_fip, db_fip) = LookupPath::new(&opctx, &datastore) + .project_id(saga_params.project_id) + .floating_ip_name(&floating_ip_name) + .fetch_for(authz::Action::Modify) + .await + .map_err(ActionError::action_failed)?; + + datastore + .floating_ip_attach(&opctx, &authz_fip, &db_fip, instance_id) + .await + .map_err(ActionError::action_failed)?; + } + } Ok(()) } @@ -638,16 +658,31 @@ async fn sic_allocate_instance_external_ip_undo( let repeat_saga_params = sagactx.saga_params::()?; let saga_params = repeat_saga_params.saga_params; let ip_index = repeat_saga_params.which; - if ip_index >= saga_params.create_params.external_ips.len() { - return Ok(()); - } - let opctx = crate::context::op_context_for_saga_action( &sagactx, &saga_params.serialized_authn, ); - let ip_id = repeat_saga_params.new_id; - datastore.deallocate_external_ip(&opctx, ip_id).await?; + let Some(ip_params) = saga_params.create_params.external_ips.get(ip_index) + else { + return Ok(()); + }; + + match ip_params { + params::ExternalIpCreate::Ephemeral { .. } => { + let ip_id = repeat_saga_params.new_id; + datastore.deallocate_external_ip(&opctx, ip_id).await?; + } + params::ExternalIpCreate::Floating { floating_ip_name } => { + let floating_ip_name = db::model::Name(floating_ip_name.clone()); + let (.., authz_fip, db_fip) = LookupPath::new(&opctx, &datastore) + .project_id(saga_params.project_id) + .floating_ip_name(&floating_ip_name) + .fetch_for(authz::Action::Modify) + .await?; + + datastore.floating_ip_detach(&opctx, &authz_fip, &db_fip).await?; + } + } Ok(()) } @@ -866,9 +901,7 @@ pub mod test { app::sagas::instance_create::SagaInstanceCreate, app::sagas::test_helpers, external_api::params, }; - use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use diesel::{ BoolExpressionMethods, ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, @@ -1013,30 +1046,28 @@ pub mod test { use nexus_db_queries::db::model::SledResource; use nexus_db_queries::db::schema::sled_resource::dsl; + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper( + "no_sled_resource_instance_records_exist", + ) + .transaction(&conn, |conn| async move { conn.batch_execute_async( nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL, ) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( - dsl::sled_resource - .filter( - dsl::kind.eq( - nexus_db_queries::db::model::SledResourceKind::Instance, - ), - ) - .select(SledResource::as_select()) - .get_results_async::(&conn) - .await - .unwrap() - .is_empty(), - ) + Ok(dsl::sled_resource + .filter(dsl::kind.eq( + nexus_db_queries::db::model::SledResourceKind::Instance, + )) + .select(SledResource::as_select()) + .get_results_async::(&conn) + .await + .unwrap() + .is_empty()) }) .await .unwrap() @@ -1048,16 +1079,17 @@ pub mod test { use nexus_db_queries::db::model::VirtualProvisioningResource; use nexus_db_queries::db::schema::virtual_provisioning_resource::dsl; - datastore.pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + let conn = datastore.pool_connection_for_tests().await.unwrap(); + + datastore + .transaction_retry_wrapper("no_virtual_provisioning_resource_records_exist") + .transaction(&conn, |conn| async move { conn .batch_execute_async(nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( + Ok( dsl::virtual_provisioning_resource .filter(dsl::resource_type.eq(nexus_db_queries::db::model::ResourceTypeProvisioned::Instance.to_string())) .select(VirtualProvisioningResource::as_select()) @@ -1075,31 +1107,29 @@ pub mod test { use nexus_db_queries::db::model::VirtualProvisioningCollection; use nexus_db_queries::db::schema::virtual_provisioning_collection::dsl; + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper( + "no_virtual_provisioning_collection_records_using_instances", + ) + .transaction(&conn, |conn| async move { conn.batch_execute_async( nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL, ) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( - dsl::virtual_provisioning_collection - .filter( - dsl::cpus_provisioned - .ne(0) - .or(dsl::ram_provisioned.ne(0)), - ) - .select(VirtualProvisioningCollection::as_select()) - .get_results_async::( - &conn, - ) - .await - .unwrap() - .is_empty(), - ) + Ok(dsl::virtual_provisioning_collection + .filter( + dsl::cpus_provisioned + .ne(0) + .or(dsl::ram_provisioned.ne(0)), + ) + .select(VirtualProvisioningCollection::as_select()) + .get_results_async::(&conn) + .await + .unwrap() + .is_empty()) }) .await .unwrap() diff --git a/nexus/src/app/sagas/instance_delete.rs b/nexus/src/app/sagas/instance_delete.rs index 1605465c74..7802312b10 100644 --- a/nexus/src/app/sagas/instance_delete.rs +++ b/nexus/src/app/sagas/instance_delete.rs @@ -8,6 +8,7 @@ use super::ActionRegistry; use super::NexusActionContext; use super::NexusSaga; use crate::app::sagas::declare_saga_actions; +use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::{authn, authz, db}; use omicron_common::api::external::{Error, ResourceType}; use omicron_common::api::internal::shared::SwitchLocation; @@ -39,6 +40,9 @@ declare_saga_actions! { DEALLOCATE_EXTERNAL_IP -> "no_result3" { + sid_deallocate_external_ip } + INSTANCE_DELETE_NAT -> "no_result4" { + + sid_delete_nat + } } // instance delete saga: definition @@ -57,6 +61,7 @@ impl NexusSaga for SagaInstanceDelete { _params: &Self::Params, mut builder: steno::DagBuilder, ) -> Result { + builder.append(instance_delete_nat_action()); builder.append(instance_delete_record_action()); builder.append(delete_network_interfaces_action()); builder.append(deallocate_external_ip_action()); @@ -110,6 +115,32 @@ async fn sid_delete_network_interfaces( Ok(()) } +async fn sid_delete_nat( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let params = sagactx.saga_params::()?; + let instance_id = params.authz_instance.id(); + let osagactx = sagactx.user_data(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let (.., authz_instance) = LookupPath::new(&opctx, &osagactx.datastore()) + .instance_id(instance_id) + .lookup_for(authz::Action::Modify) + .await + .map_err(ActionError::action_failed)?; + + osagactx + .nexus() + .instance_delete_dpd_config(&opctx, &authz_instance) + .await + .map_err(ActionError::action_failed)?; + + Ok(()) +} + async fn sid_deallocate_external_ip( sagactx: NexusActionContext, ) -> Result<(), ActionError> { @@ -127,6 +158,11 @@ async fn sid_deallocate_external_ip( ) .await .map_err(ActionError::action_failed)?; + osagactx + .datastore() + .detach_floating_ips_by_instance_id(&opctx, params.authz_instance.id()) + .await + .map_err(ActionError::action_failed)?; Ok(()) } diff --git a/nexus/src/app/sagas/project_create.rs b/nexus/src/app/sagas/project_create.rs index 135e20ff06..40acc822c0 100644 --- a/nexus/src/app/sagas/project_create.rs +++ b/nexus/src/app/sagas/project_create.rs @@ -157,9 +157,7 @@ mod test { app::saga::create_saga_dag, app::sagas::project_create::Params, app::sagas::project_create::SagaProjectCreate, external_api::params, }; - use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use diesel::{ ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, }; @@ -233,15 +231,16 @@ mod test { use nexus_db_queries::db::model::VirtualProvisioningCollection; use nexus_db_queries::db::schema::virtual_provisioning_collection::dsl; - datastore.pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + let conn = datastore.pool_connection_for_tests().await.unwrap(); + + datastore + .transaction_retry_wrapper("no_virtual_provisioning_collection_records_for_projects") + .transaction(&conn, |conn| async move { conn .batch_execute_async(nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( + Ok( dsl::virtual_provisioning_collection .filter(dsl::collection_type.eq(nexus_db_queries::db::model::CollectionTypeProvisioned::Project.to_string())) // ignore built-in services project diff --git a/nexus/src/app/sagas/switch_port_settings_common.rs b/nexus/src/app/sagas/switch_port_settings_common.rs index 9132645782..9ef23ebf44 100644 --- a/nexus/src/app/sagas/switch_port_settings_common.rs +++ b/nexus/src/app/sagas/switch_port_settings_common.rs @@ -444,7 +444,9 @@ pub(crate) async fn ensure_switch_port_bgp_settings( |e| ActionError::action_failed(format!("select mg client: {e}")), )?; - let mut bgp_peer_configs = Vec::new(); + let mut bgp_peer_configs = HashMap::>::new(); + + let mut cfg: Option = None; for peer in settings.bgp_peers { let config = nexus @@ -454,11 +456,44 @@ pub(crate) async fn ensure_switch_port_bgp_settings( ActionError::action_failed(format!("get bgp config: {e}")) })?; + if let Some(cfg) = &cfg { + if config.asn != cfg.asn { + return Err(ActionError::action_failed( + "bad request: only one AS allowed per switch".to_string(), + )); + } + } else { + cfg = Some(config); + } + + let bpc = BgpPeerConfig { + name: format!("{}", peer.addr.ip()), //TODO user defined name? + host: format!("{}:179", peer.addr.ip()), + hold_time: peer.hold_time.0.into(), + idle_hold_time: peer.idle_hold_time.0.into(), + delay_open: peer.delay_open.0.into(), + connect_retry: peer.connect_retry.0.into(), + keepalive: peer.keepalive.0.into(), + resolution: BGP_SESSION_RESOLUTION, + passive: false, + }; + + match bgp_peer_configs.get_mut(&switch_port_name) { + Some(peers) => { + peers.push(bpc); + } + None => { + bgp_peer_configs.insert(switch_port_name.clone(), vec![bpc]); + } + } + } + + if let Some(cfg) = &cfg { let announcements = nexus .bgp_announce_list( &opctx, ¶ms::BgpAnnounceSetSelector { - name_or_id: NameOrId::Id(config.bgp_announce_set_id), + name_or_id: NameOrId::Id(cfg.bgp_announce_set_id), }, ) .await @@ -473,39 +508,25 @@ pub(crate) async fn ensure_switch_port_bgp_settings( let value = match a.network.ip() { IpAddr::V4(value) => Ok(value), IpAddr::V6(_) => Err(ActionError::action_failed( - "IPv6 announcement not yet supported".to_string(), + "bad request: IPv6 announcement not yet supported" + .to_string(), )), }?; prefixes.push(Prefix4 { value, length: a.network.prefix() }); } - - let bpc = BgpPeerConfig { - asn: *config.asn, - name: format!("{}", peer.addr.ip()), //TODO user defined name? - host: format!("{}:179", peer.addr.ip()), - hold_time: peer.hold_time.0.into(), - idle_hold_time: peer.idle_hold_time.0.into(), - delay_open: peer.delay_open.0.into(), - connect_retry: peer.connect_retry.0.into(), - keepalive: peer.keepalive.0.into(), - resolution: BGP_SESSION_RESOLUTION, - originate: prefixes, - }; - - bgp_peer_configs.push(bpc); + mg_client + .inner + .bgp_apply(&ApplyRequest { + asn: cfg.asn.0, + peers: bgp_peer_configs, + originate: prefixes, + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("apply bgp settings: {e}")) + })?; } - mg_client - .inner - .bgp_apply(&ApplyRequest { - peer_group: switch_port_name, - peers: bgp_peer_configs, - }) - .await - .map_err(|e| { - ActionError::action_failed(format!("apply bgp settings: {e}")) - })?; - Ok(()) } diff --git a/nexus/src/app/sagas/test_helpers.rs b/nexus/src/app/sagas/test_helpers.rs index eccb013b66..3110bd318a 100644 --- a/nexus/src/app/sagas/test_helpers.rs +++ b/nexus/src/app/sagas/test_helpers.rs @@ -10,9 +10,7 @@ use crate::{ app::{saga::create_saga_dag, test_interfaces::TestInterfaces as _}, Nexus, }; -use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, -}; +use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use futures::future::BoxFuture; use nexus_db_queries::{ @@ -434,11 +432,10 @@ pub(crate) async fn assert_no_failed_undo_steps( ) { use nexus_db_queries::db::model::saga_types::SagaNodeEvent; + let conn = datastore.pool_connection_for_tests().await.unwrap(); let saga_node_events: Vec = datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper("assert_no_failed_undo_steps") + .transaction(&conn, |conn| async move { use nexus_db_queries::db::schema::saga_node_event::dsl; conn.batch_execute_async( @@ -447,14 +444,12 @@ pub(crate) async fn assert_no_failed_undo_steps( .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( - dsl::saga_node_event - .filter(dsl::event_type.eq(String::from("undo_failed"))) - .select(SagaNodeEvent::as_select()) - .load_async::(&conn) - .await - .unwrap(), - ) + Ok(dsl::saga_node_event + .filter(dsl::event_type.eq(String::from("undo_failed"))) + .select(SagaNodeEvent::as_select()) + .load_async::(&conn) + .await + .unwrap()) }) .await .unwrap(); diff --git a/nexus/src/app/switch_port.rs b/nexus/src/app/switch_port.rs index acc57459fd..b9f0f94fa0 100644 --- a/nexus/src/app/switch_port.rs +++ b/nexus/src/app/switch_port.rs @@ -117,7 +117,6 @@ impl super::Nexus { .map_err(|e| { let msg = e.to_string(); if msg.contains("bad request") { - //return HttpError::for_client_error(None, StatusCode::BAD_REQUEST, msg.to_string()) external::Error::invalid_request(&msg.to_string()) } else { e @@ -255,7 +254,15 @@ impl super::Nexus { >( saga_params, ) - .await?; + .await + .map_err(|e| { + let msg = e.to_string(); + if msg.contains("bad request") { + external::Error::invalid_request(&msg.to_string()) + } else { + e + } + })?; Ok(()) } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 9ac1f22501..a2e5f633df 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -140,6 +140,11 @@ pub(crate) fn external_api() -> NexusApiDescription { api.register(ip_pool_service_range_add)?; api.register(ip_pool_service_range_remove)?; + api.register(floating_ip_list)?; + api.register(floating_ip_create)?; + api.register(floating_ip_view)?; + api.register(floating_ip_delete)?; + api.register(disk_list)?; api.register(disk_create)?; api.register(disk_view)?; @@ -1520,6 +1525,126 @@ async fn ip_pool_service_range_remove( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } +// Floating IP Addresses + +/// List all Floating IPs +#[endpoint { + method = GET, + path = "/v1/floating-ips", + tags = ["floating-ips"], +}] +async fn floating_ip_list( + rqctx: RequestContext>, + query_params: Query>, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let project_lookup = + nexus.project_lookup(&opctx, scan_params.selector.clone())?; + let ips = nexus + .floating_ips_list(&opctx, &project_lookup, &paginated_by) + .await?; + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + ips, + &marker_for_name_or_id, + )?)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Create a Floating IP +#[endpoint { + method = POST, + path = "/v1/floating-ips", + tags = ["floating-ips"], +}] +async fn floating_ip_create( + rqctx: RequestContext>, + query_params: Query, + floating_params: TypedBody, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let nexus = &apictx.nexus; + let floating_params = floating_params.into_inner(); + let handler = async { + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let project_lookup = + nexus.project_lookup(&opctx, query_params.into_inner())?; + let ip = nexus + .floating_ip_create(&opctx, &project_lookup, floating_params) + .await?; + Ok(HttpResponseCreated(ip)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Delete a Floating IP +#[endpoint { + method = DELETE, + path = "/v1/floating-ips/{floating_ip}", + tags = ["floating-ips"], +}] +async fn floating_ip_delete( + rqctx: RequestContext>, + path_params: Path, + query_params: Query, +) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let floating_ip_selector = params::FloatingIpSelector { + floating_ip: path.floating_ip, + project: query.project, + }; + let fip_lookup = + nexus.floating_ip_lookup(&opctx, floating_ip_selector)?; + + nexus.floating_ip_delete(&opctx, fip_lookup).await?; + Ok(HttpResponseDeleted()) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Fetch a floating IP +#[endpoint { + method = GET, + path = "/v1/floating-ips/{floating_ip}", + tags = ["floating-ips"] +}] +async fn floating_ip_view( + rqctx: RequestContext>, + path_params: Path, + query_params: Query, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let floating_ip_selector = params::FloatingIpSelector { + floating_ip: path.floating_ip, + project: query.project, + }; + let (.., fip) = nexus + .floating_ip_lookup(&opctx, floating_ip_selector)? + .fetch() + .await?; + Ok(HttpResponseOk(fip.into())) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + // Disks /// List disks diff --git a/nexus/src/external_api/tag-config.json b/nexus/src/external_api/tag-config.json index 07eb198016..3bc8006cee 100644 --- a/nexus/src/external_api/tag-config.json +++ b/nexus/src/external_api/tag-config.json @@ -8,6 +8,12 @@ "url": "http://docs.oxide.computer/api/disks" } }, + "floating-ips": { + "description": "Floating IPs allow a project to allocate well-known IPs to instances.", + "external_docs": { + "url": "http://docs.oxide.computer/api/floating-ips" + } + }, "hidden": { "description": "TODO operations that will not ship to customers", "external_docs": { diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 2368c3f568..1848989bf9 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -21,6 +21,7 @@ use nexus_types::external_api::shared::IdentityType; use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::views; use nexus_types::external_api::views::Certificate; +use nexus_types::external_api::views::FloatingIp; use nexus_types::external_api::views::IpPool; use nexus_types::external_api::views::IpPoolRange; use nexus_types::external_api::views::User; @@ -32,7 +33,9 @@ use omicron_common::api::external::Disk; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::Instance; use omicron_common::api::external::InstanceCpuCount; +use omicron_common::api::external::NameOrId; use omicron_sled_agent::sim::SledAgent; +use std::net::IpAddr; use std::sync::Arc; use uuid::Uuid; @@ -149,6 +152,28 @@ pub async fn create_ip_pool( (pool, range) } +pub async fn create_floating_ip( + client: &ClientTestContext, + fip_name: &str, + project: &str, + address: Option, + parent_pool_name: Option<&str>, +) -> FloatingIp { + object_create( + client, + &format!("/v1/floating-ips?project={project}"), + ¶ms::FloatingIpCreate { + identity: IdentityMetadataCreateParams { + name: fip_name.parse().unwrap(), + description: String::from("a floating ip"), + }, + address, + pool: parent_pool_name.map(|v| NameOrId::Name(v.parse().unwrap())), + }, + ) + .await +} + pub async fn create_certificate( client: &ClientTestContext, cert_name: &str, diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index f693d36ec4..e11902d0fe 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -134,6 +134,7 @@ lazy_static! { pub static ref DEMO_PROJECT_URL_INSTANCES: String = format!("/v1/instances?project={}", *DEMO_PROJECT_NAME); pub static ref DEMO_PROJECT_URL_SNAPSHOTS: String = format!("/v1/snapshots?project={}", *DEMO_PROJECT_NAME); pub static ref DEMO_PROJECT_URL_VPCS: String = format!("/v1/vpcs?project={}", *DEMO_PROJECT_NAME); + pub static ref DEMO_PROJECT_URL_FIPS: String = format!("/v1/floating-ips?project={}", *DEMO_PROJECT_NAME); pub static ref DEMO_PROJECT_CREATE: params::ProjectCreate = params::ProjectCreate { identity: IdentityMetadataCreateParams { @@ -568,6 +569,22 @@ lazy_static! { }; } +lazy_static! { + // Project Floating IPs + pub static ref DEMO_FLOAT_IP_NAME: Name = "float-ip".parse().unwrap(); + pub static ref DEMO_FLOAT_IP_URL: String = + format!("/v1/floating-ips/{}?project={}", *DEMO_FLOAT_IP_NAME, *DEMO_PROJECT_NAME); + pub static ref DEMO_FLOAT_IP_CREATE: params::FloatingIpCreate = + params::FloatingIpCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_FLOAT_IP_NAME.clone(), + description: String::from("a new IP pool"), + }, + address: Some(std::net::Ipv4Addr::new(10, 0, 0, 141).into()), + pool: None, + }; +} + lazy_static! { // Identity providers pub static ref IDENTITY_PROVIDERS_URL: String = format!("/v1/system/identity-providers?silo=demo-silo"); @@ -1972,6 +1989,29 @@ lazy_static! { allowed_methods: vec![ AllowedMethod::GetNonexistent, ], + }, + + // Floating IPs + VerifyEndpoint { + url: &DEMO_PROJECT_URL_FIPS, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_FLOAT_IP_CREATE).unwrap(), + ), + AllowedMethod::Get, + ], + }, + + VerifyEndpoint { + url: &DEMO_FLOAT_IP_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Delete, + ], } ]; } diff --git a/nexus/tests/integration_tests/external_ips.rs b/nexus/tests/integration_tests/external_ips.rs new file mode 100644 index 0000000000..f3161dea72 --- /dev/null +++ b/nexus/tests/integration_tests/external_ips.rs @@ -0,0 +1,432 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tests Floating IP support in the API + +use std::net::IpAddr; +use std::net::Ipv4Addr; + +use crate::integration_tests::instances::instance_simulate; +use dropshot::test_util::ClientTestContext; +use dropshot::HttpErrorResponseBody; +use http::Method; +use http::StatusCode; +use nexus_test_utils::http_testing::AuthnMode; +use nexus_test_utils::http_testing::NexusRequest; +use nexus_test_utils::http_testing::RequestBuilder; +use nexus_test_utils::resource_helpers::create_floating_ip; +use nexus_test_utils::resource_helpers::create_instance_with; +use nexus_test_utils::resource_helpers::create_ip_pool; +use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils::resource_helpers::populate_ip_pool; +use nexus_test_utils_macros::nexus_test; +use nexus_types::external_api::params; +use nexus_types::external_api::views::FloatingIp; +use omicron_common::address::IpRange; +use omicron_common::address::Ipv4Range; +use omicron_common::api::external::IdentityMetadataCreateParams; +use omicron_common::api::external::Instance; +use uuid::Uuid; + +type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + +const PROJECT_NAME: &str = "rootbeer-float"; + +const FIP_NAMES: &[&str] = &["vanilla", "chocolate", "strawberry", "pistachio"]; + +pub fn get_floating_ips_url(project_name: &str) -> String { + format!("/v1/floating-ips?project={project_name}") +} + +pub fn get_floating_ip_by_name_url( + fip_name: &str, + project_name: &str, +) -> String { + format!("/v1/floating-ips/{fip_name}?project={project_name}") +} + +pub fn get_floating_ip_by_id_url(fip_id: &Uuid) -> String { + format!("/v1/floating-ips/{fip_id}") +} + +#[nexus_test] +async fn test_floating_ip_access(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + + populate_ip_pool(&client, "default", None).await; + let project = create_project(client, PROJECT_NAME).await; + + // Create a floating IP from the default pool. + let fip_name = FIP_NAMES[0]; + let fip = create_floating_ip( + client, + fip_name, + &project.identity.id.to_string(), + None, + None, + ) + .await; + + // Fetch floating IP by ID + let fetched_fip = + floating_ip_get(&client, &get_floating_ip_by_id_url(&fip.identity.id)) + .await; + assert_eq!(fetched_fip.identity.id, fip.identity.id); + + // Fetch floating IP by name and project_id + let fetched_fip = floating_ip_get( + &client, + &get_floating_ip_by_name_url( + fip.identity.name.as_str(), + &project.identity.id.to_string(), + ), + ) + .await; + assert_eq!(fetched_fip.identity.id, fip.identity.id); + + // Fetch floating IP by name and project_name + let fetched_fip = floating_ip_get( + &client, + &get_floating_ip_by_name_url( + fip.identity.name.as_str(), + project.identity.name.as_str(), + ), + ) + .await; + assert_eq!(fetched_fip.identity.id, fip.identity.id); +} + +#[nexus_test] +async fn test_floating_ip_create(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + + populate_ip_pool(&client, "default", None).await; + let other_pool_range = IpRange::V4( + Ipv4Range::new(Ipv4Addr::new(10, 1, 0, 1), Ipv4Addr::new(10, 1, 0, 5)) + .unwrap(), + ); + create_ip_pool(&client, "other-pool", Some(other_pool_range)).await; + + let project = create_project(client, PROJECT_NAME).await; + + // Create with no chosen IP and fallback to default pool. + let fip_name = FIP_NAMES[0]; + let fip = create_floating_ip( + client, + fip_name, + project.identity.name.as_str(), + None, + None, + ) + .await; + assert_eq!(fip.identity.name.as_str(), fip_name); + assert_eq!(fip.project_id, project.identity.id); + assert_eq!(fip.instance_id, None); + assert_eq!(fip.ip, IpAddr::from(Ipv4Addr::new(10, 0, 0, 0))); + + // Create with chosen IP and fallback to default pool. + let fip_name = FIP_NAMES[1]; + let ip_addr = "10.0.12.34".parse().unwrap(); + let fip = create_floating_ip( + client, + fip_name, + project.identity.name.as_str(), + Some(ip_addr), + None, + ) + .await; + assert_eq!(fip.identity.name.as_str(), fip_name); + assert_eq!(fip.project_id, project.identity.id); + assert_eq!(fip.instance_id, None); + assert_eq!(fip.ip, ip_addr); + + // Create with no chosen IP from named pool. + let fip_name = FIP_NAMES[2]; + let fip = create_floating_ip( + client, + fip_name, + project.identity.name.as_str(), + None, + Some("other-pool"), + ) + .await; + assert_eq!(fip.identity.name.as_str(), fip_name); + assert_eq!(fip.project_id, project.identity.id); + assert_eq!(fip.instance_id, None); + assert_eq!(fip.ip, IpAddr::from(Ipv4Addr::new(10, 1, 0, 1))); + + // Create with chosen IP from named pool. + let fip_name = FIP_NAMES[3]; + let ip_addr = "10.1.0.5".parse().unwrap(); + let fip = create_floating_ip( + client, + fip_name, + project.identity.name.as_str(), + Some(ip_addr), + Some("other-pool"), + ) + .await; + assert_eq!(fip.identity.name.as_str(), fip_name); + assert_eq!(fip.project_id, project.identity.id); + assert_eq!(fip.instance_id, None); + assert_eq!(fip.ip, ip_addr); +} + +#[nexus_test] +async fn test_floating_ip_create_ip_in_use( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + populate_ip_pool(&client, "default", None).await; + + let project = create_project(client, PROJECT_NAME).await; + let contested_ip = "10.0.0.0".parse().unwrap(); + + // First create will succeed. + create_floating_ip( + client, + FIP_NAMES[0], + project.identity.name.as_str(), + Some(contested_ip), + None, + ) + .await; + + // Second will fail as the requested IP is in use in the selected + // (default) pool. + let error: HttpErrorResponseBody = NexusRequest::new( + RequestBuilder::new( + client, + Method::POST, + &get_floating_ips_url(PROJECT_NAME), + ) + .body(Some(¶ms::FloatingIpCreate { + identity: IdentityMetadataCreateParams { + name: FIP_NAMES[1].parse().unwrap(), + description: "another fip".into(), + }, + address: Some(contested_ip), + pool: None, + })) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + assert_eq!(error.message, "Requested external IP address not available"); +} + +#[nexus_test] +async fn test_floating_ip_create_name_in_use( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + populate_ip_pool(&client, "default", None).await; + + let project = create_project(client, PROJECT_NAME).await; + let contested_name = FIP_NAMES[0]; + + // First create will succeed. + create_floating_ip( + client, + contested_name, + project.identity.name.as_str(), + None, + None, + ) + .await; + + // Second will fail as the requested name is in use within this + // project. + let error: HttpErrorResponseBody = NexusRequest::new( + RequestBuilder::new( + client, + Method::POST, + &get_floating_ips_url(PROJECT_NAME), + ) + .body(Some(¶ms::FloatingIpCreate { + identity: IdentityMetadataCreateParams { + name: contested_name.parse().unwrap(), + description: "another fip".into(), + }, + address: None, + pool: None, + })) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + assert_eq!( + error.message, + format!("already exists: floating-ip \"{contested_name}\""), + ); +} + +#[nexus_test] +async fn test_floating_ip_delete(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + + populate_ip_pool(&client, "default", None).await; + let project = create_project(client, PROJECT_NAME).await; + + let fip = create_floating_ip( + client, + FIP_NAMES[0], + project.identity.name.as_str(), + None, + None, + ) + .await; + + // Delete the floating IP. + NexusRequest::object_delete( + client, + &get_floating_ip_by_id_url(&fip.identity.id), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); +} + +#[nexus_test] +async fn test_floating_ip_attachment(cptestctx: &ControlPlaneTestContext) { + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + + populate_ip_pool(&client, "default", None).await; + let project = create_project(client, PROJECT_NAME).await; + + let fip = create_floating_ip( + client, + FIP_NAMES[0], + project.identity.name.as_str(), + None, + None, + ) + .await; + + // Bind the floating IP to an instance at create time. + let instance_name = "anonymous-diner"; + let instance = create_instance_with( + &client, + PROJECT_NAME, + instance_name, + ¶ms::InstanceNetworkInterfaceAttachment::Default, + vec![], + vec![params::ExternalIpCreate::Floating { + floating_ip_name: FIP_NAMES[0].parse().unwrap(), + }], + ) + .await; + + // Reacquire FIP: parent ID must have updated to match instance. + let fetched_fip = + floating_ip_get(&client, &get_floating_ip_by_id_url(&fip.identity.id)) + .await; + assert_eq!(fetched_fip.instance_id, Some(instance.identity.id)); + + // Try to delete the floating IP, which should fail. + let error: HttpErrorResponseBody = NexusRequest::new( + RequestBuilder::new( + client, + Method::DELETE, + &get_floating_ip_by_id_url(&fip.identity.id), + ) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + assert_eq!( + error.message, + format!("Floating IP cannot be deleted while attached to an instance"), + ); + + // Stop and delete the instance. + instance_simulate(nexus, &instance.identity.id).await; + instance_simulate(nexus, &instance.identity.id).await; + + let _: Instance = NexusRequest::new( + RequestBuilder::new( + client, + Method::POST, + &format!("/v1/instances/{}/stop", instance.identity.id), + ) + .body(None as Option<&serde_json::Value>) + .expect_status(Some(StatusCode::ACCEPTED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + + instance_simulate(nexus, &instance.identity.id).await; + + NexusRequest::object_delete( + &client, + &format!("/v1/instances/{instance_name}?project={PROJECT_NAME}"), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + + // Reacquire FIP again: parent ID must now be unset. + let fetched_fip = + floating_ip_get(&client, &get_floating_ip_by_id_url(&fip.identity.id)) + .await; + assert_eq!(fetched_fip.instance_id, None); + + // Delete the floating IP. + NexusRequest::object_delete( + client, + &get_floating_ip_by_id_url(&fip.identity.id), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); +} + +pub async fn floating_ip_get( + client: &ClientTestContext, + fip_url: &str, +) -> FloatingIp { + floating_ip_get_as(client, fip_url, AuthnMode::PrivilegedUser).await +} + +async fn floating_ip_get_as( + client: &ClientTestContext, + fip_url: &str, + authn_as: AuthnMode, +) -> FloatingIp { + NexusRequest::object_get(client, fip_url) + .authn_as(authn_as) + .execute() + .await + .unwrap_or_else(|e| { + panic!("failed to make \"get\" request to {fip_url}: {e}") + }) + .parsed_body() + .unwrap_or_else(|e| { + panic!("failed to make \"get\" request to {fip_url}: {e}") + }) +} diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index eaae35afff..33d4d15d23 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -4,11 +4,14 @@ //! Tests basic instance support in the API +use super::external_ips::floating_ip_get; +use super::external_ips::get_floating_ip_by_id_url; use super::metrics::{get_latest_silo_metric, get_latest_system_metric}; use camino::Utf8Path; use http::method::Method; use http::StatusCode; +use itertools::Itertools; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_db_queries::db::fixed_data::silo::SILO_ID; @@ -18,6 +21,7 @@ use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::resource_helpers::create_disk; +use nexus_test_utils::resource_helpers::create_floating_ip; use nexus_test_utils::resource_helpers::create_ip_pool; use nexus_test_utils::resource_helpers::create_local_user; use nexus_test_utils::resource_helpers::create_silo; @@ -54,6 +58,7 @@ use omicron_nexus::TestInterfaces as _; use omicron_sled_agent::sim::SledAgent; use sled_agent_client::TestInterfaces as _; use std::convert::TryFrom; +use std::net::Ipv4Addr; use std::sync::Arc; use uuid::Uuid; @@ -3628,6 +3633,139 @@ async fn test_instance_ephemeral_ip_from_correct_pool( ); } +#[nexus_test] +async fn test_instance_attach_several_external_ips( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + let _ = create_project(&client, PROJECT_NAME).await; + + // Create a single (large) IP pool + let default_pool_range = IpRange::V4( + Ipv4Range::new( + std::net::Ipv4Addr::new(10, 0, 0, 1), + std::net::Ipv4Addr::new(10, 0, 0, 10), + ) + .unwrap(), + ); + populate_ip_pool(&client, "default", Some(default_pool_range)).await; + + // Create several floating IPs for the instance, totalling 8 IPs. + let mut external_ip_create = + vec![params::ExternalIpCreate::Ephemeral { pool_name: None }]; + let mut fips = vec![]; + for i in 1..8 { + let name = format!("fip-{i}"); + fips.push( + create_floating_ip(&client, &name, PROJECT_NAME, None, None).await, + ); + external_ip_create.push(params::ExternalIpCreate::Floating { + floating_ip_name: name.parse().unwrap(), + }); + } + + // Create an instance with pool name blank, expect IP from default pool + let instance_name = "many-fips"; + let instance = create_instance_with( + &client, + PROJECT_NAME, + instance_name, + ¶ms::InstanceNetworkInterfaceAttachment::Default, + vec![], + external_ip_create, + ) + .await; + + // Verify that all external IPs are visible on the instance and have + // been allocated in order. + let external_ips = + fetch_instance_external_ips(&client, instance_name).await; + assert_eq!(external_ips.len(), 8); + eprintln!("{external_ips:?}"); + for (i, eip) in external_ips + .iter() + .sorted_unstable_by(|a, b| a.ip.cmp(&b.ip)) + .enumerate() + { + let last_octet = i + if i != external_ips.len() - 1 { + assert_eq!(eip.kind, IpKind::Floating); + 1 + } else { + // SNAT will occupy 1.0.0.8 here, since it it alloc'd before + // the ephemeral. + assert_eq!(eip.kind, IpKind::Ephemeral); + 2 + }; + assert_eq!(eip.ip, Ipv4Addr::new(10, 0, 0, last_octet as u8)); + } + + // Verify that all floating IPs are bound to their parent instance. + for fip in fips { + let fetched_fip = floating_ip_get( + &client, + &get_floating_ip_by_id_url(&fip.identity.id), + ) + .await; + assert_eq!(fetched_fip.instance_id, Some(instance.identity.id)); + } +} + +#[nexus_test] +async fn test_instance_allow_only_one_ephemeral_ip( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + + let _ = create_project(&client, PROJECT_NAME).await; + + // Create one IP pool with space for two ephemerals. + let default_pool_range = IpRange::V4( + Ipv4Range::new( + std::net::Ipv4Addr::new(10, 0, 0, 1), + std::net::Ipv4Addr::new(10, 0, 0, 2), + ) + .unwrap(), + ); + populate_ip_pool(&client, "default", Some(default_pool_range)).await; + + let ephemeral_create = params::ExternalIpCreate::Ephemeral { + pool_name: Some("default".parse().unwrap()), + }; + let error: HttpErrorResponseBody = NexusRequest::new( + RequestBuilder::new(client, Method::POST, &get_instances_url()) + .body(Some(¶ms::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: "default-pool-inst".parse().unwrap(), + description: "instance default-pool-inst".into(), + }, + ncpus: InstanceCpuCount(4), + memory: ByteCount::from_gibibytes_u32(1), + hostname: String::from("the_host"), + user_data: + b"#cloud-config\nsystem_info:\n default_user:\n name: oxide" + .to_vec(), + network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, + external_ips: vec![ + ephemeral_create.clone(), ephemeral_create + ], + disks: vec![], + start: true, + })) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + assert_eq!( + error.message, + "An instance may not have more than 1 ephemeral IP address" + ); +} + async fn create_instance_with_pool( client: &ClientTestContext, instance_name: &str, @@ -3646,10 +3784,10 @@ async fn create_instance_with_pool( .await } -async fn fetch_instance_ephemeral_ip( +async fn fetch_instance_external_ips( client: &ClientTestContext, instance_name: &str, -) -> views::ExternalIp { +) -> Vec { let ips_url = format!( "/v1/instances/{}/external-ips?project={}", instance_name, PROJECT_NAME @@ -3661,9 +3799,18 @@ async fn fetch_instance_ephemeral_ip( .expect("Failed to fetch external IPs") .parsed_body::>() .expect("Failed to parse external IPs"); - assert_eq!(ips.items.len(), 1); - assert_eq!(ips.items[0].kind, IpKind::Ephemeral); - ips.items[0].clone() + ips.items +} + +async fn fetch_instance_ephemeral_ip( + client: &ClientTestContext, + instance_name: &str, +) -> views::ExternalIp { + fetch_instance_external_ips(client, instance_name) + .await + .into_iter() + .find(|v| v.kind == IpKind::Ephemeral) + .unwrap() } #[nexus_test] diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs index 4d7b41cfa8..53de24c518 100644 --- a/nexus/tests/integration_tests/mod.rs +++ b/nexus/tests/integration_tests/mod.rs @@ -12,6 +12,7 @@ mod commands; mod console_api; mod device_auth; mod disks; +mod external_ips; mod host_phase1_updater; mod images; mod initialization; diff --git a/nexus/tests/integration_tests/unauthorized.rs b/nexus/tests/integration_tests/unauthorized.rs index 9936af20bf..1cb2eaca3a 100644 --- a/nexus/tests/integration_tests/unauthorized.rs +++ b/nexus/tests/integration_tests/unauthorized.rs @@ -278,6 +278,12 @@ lazy_static! { body: serde_json::to_value(&*DEMO_IMAGE_CREATE).unwrap(), id_routes: vec!["/v1/images/{id}"], }, + // Create a Floating IP in the project + SetupReq::Post { + url: &DEMO_PROJECT_URL_FIPS, + body: serde_json::to_value(&*DEMO_FLOAT_IP_CREATE).unwrap(), + id_routes: vec!["/v1/floating-ips/{id}"], + }, // Create a SAML identity provider SetupReq::Post { url: &SAML_IDENTITY_PROVIDERS_URL, diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 14323049c2..5a4a61132e 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -10,6 +10,13 @@ disk_list GET /v1/disks disk_metrics_list GET /v1/disks/{disk}/metrics/{metric} disk_view GET /v1/disks/{disk} +API operations found with tag "floating-ips" +OPERATION ID METHOD URL PATH +floating_ip_create POST /v1/floating-ips +floating_ip_delete DELETE /v1/floating-ips/{floating_ip} +floating_ip_list GET /v1/floating-ips +floating_ip_view GET /v1/floating-ips/{floating_ip} + API operations found with tag "hidden" OPERATION ID METHOD URL PATH device_access_token POST /device/token diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index a8255fb836..cde448c5b7 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -54,6 +54,7 @@ path_param!(VpcPath, vpc, "VPC"); path_param!(SubnetPath, subnet, "subnet"); path_param!(RouterPath, router, "router"); path_param!(RoutePath, route, "route"); +path_param!(FloatingIpPath, floating_ip, "Floating IP"); path_param!(DiskPath, disk, "disk"); path_param!(SnapshotPath, snapshot, "snapshot"); path_param!(ImagePath, image, "image"); @@ -146,6 +147,14 @@ pub struct OptionalProjectSelector { pub project: Option, } +#[derive(Deserialize, JsonSchema)] +pub struct FloatingIpSelector { + /// Name or ID of the project, only required if `floating_ip` is provided as a `Name` + pub project: Option, + /// Name or ID of the Floating IP + pub floating_ip: NameOrId, +} + #[derive(Deserialize, JsonSchema)] pub struct DiskSelector { /// Name or ID of the project, only required if `disk` is provided as a `Name` @@ -768,6 +777,23 @@ pub struct IpPoolUpdate { pub identity: IdentityMetadataUpdateParams, } +// Floating IPs +/// Parameters for creating a new floating IP address for instances. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct FloatingIpCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, + + /// An IP address to reserve for use as a floating IP. This field is + /// optional: when not set, an address will be automatically chosen from + /// `pool`. If set, then the IP must be available in the resolved `pool`. + pub address: Option, + + /// The parent IP pool that a floating IP is pulled from. If unset, the + /// default pool is selected. + pub pool: Option, +} + // INSTANCES /// Describes an attachment of an `InstanceNetworkInterface` to an `Instance`, @@ -835,7 +861,11 @@ pub enum ExternalIpCreate { /// automatically-assigned from the provided IP Pool, or all available pools /// if not specified. Ephemeral { pool_name: Option }, - // TODO: Add floating IPs: https://github.com/oxidecomputer/omicron/issues/1334 + /// An IP address providing both inbound and outbound access. The address is + /// an existing Floating IP object assigned to the current project. + /// + /// The floating IP must not be in use by another instance or service. + Floating { floating_ip_name: Name }, } /// Create-time parameters for an `Instance` diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index e46a5ba5dc..af17e7e840 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -262,6 +262,22 @@ pub struct ExternalIp { pub kind: IpKind, } +/// A Floating IP is a well-known IP address which can be attached +/// and detached from instances. +#[derive(ObjectIdentity, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct FloatingIp { + #[serde(flatten)] + pub identity: IdentityMetadata, + /// The IP address held by this resource. + pub ip: IpAddr, + /// The project this resource exists within. + pub project_id: Uuid, + /// The ID of the instance that this Floating IP is attached to, + /// if it is presently in use. + pub instance_id: Option, +} + // RACKS /// View of an Rack diff --git a/openapi/nexus.json b/openapi/nexus.json index 619bacc602..7afb6cdc2f 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -804,6 +804,204 @@ } } }, + "/v1/floating-ips": { + "get": { + "tags": [ + "floating-ips" + ], + "summary": "List all Floating IPs", + "operationId": "floating_ip_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIpResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "floating-ips" + ], + "summary": "Create a Floating IP", + "operationId": "floating_ip_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIpCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIp" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/floating-ips/{floating_ip}": { + "get": { + "tags": [ + "floating-ips" + ], + "summary": "Fetch a floating IP", + "operationId": "floating_ip_view", + "parameters": [ + { + "in": "path", + "name": "floating_ip", + "description": "Name or ID of the Floating IP", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FloatingIp" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "floating-ips" + ], + "summary": "Delete a Floating IP", + "operationId": "floating_ip_delete", + "parameters": [ + { + "in": "path", + "name": "floating_ip", + "description": "Name or ID of the Floating IP", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/groups": { "get": { "tags": [ @@ -10321,6 +10519,25 @@ "required": [ "type" ] + }, + { + "description": "An IP address providing both inbound and outbound access. The address is an existing Floating IP object assigned to the current project.\n\nThe floating IP must not be in use by another instance or service.", + "type": "object", + "properties": { + "floating_ip_name": { + "$ref": "#/components/schemas/Name" + }, + "type": { + "type": "string", + "enum": [ + "floating" + ] + } + }, + "required": [ + "floating_ip_name", + "type" + ] } ] }, @@ -10405,6 +10622,116 @@ "role_name" ] }, + "FloatingIp": { + "description": "A Floating IP is a well-known IP address which can be attached and detached from instances.", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "instance_id": { + "nullable": true, + "description": "The ID of the instance that this Floating IP is attached to, if it is presently in use.", + "type": "string", + "format": "uuid" + }, + "ip": { + "description": "The IP address held by this resource.", + "type": "string", + "format": "ip" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "project_id": { + "description": "The project this resource exists within.", + "type": "string", + "format": "uuid" + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "id", + "ip", + "name", + "project_id", + "time_created", + "time_modified" + ] + }, + "FloatingIpCreate": { + "description": "Parameters for creating a new floating IP address for instances.", + "type": "object", + "properties": { + "address": { + "nullable": true, + "description": "An IP address to reserve for use as a floating IP. This field is optional: when not set, an address will be automatically chosen from `pool`. If set, then the IP must be available in the resolved `pool`.", + "type": "string", + "format": "ip" + }, + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "pool": { + "nullable": true, + "description": "The parent IP pool that a floating IP is pulled from. If unset, the default pool is selected.", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + } + }, + "required": [ + "description", + "name" + ] + }, + "FloatingIpResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/FloatingIp" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, "Group": { "description": "View of a Group", "type": "object", @@ -15147,6 +15474,13 @@ "url": "http://docs.oxide.computer/api/disks" } }, + { + "name": "floating-ips", + "description": "Floating IPs allow a project to allocate well-known IPs to instances.", + "externalDocs": { + "url": "http://docs.oxide.computer/api/floating-ips" + } + }, { "name": "hidden", "description": "TODO operations that will not ship to customers", diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 5e217b27a4..3a88b6cc9c 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -4252,18 +4252,23 @@ "$ref": "#/components/schemas/DiskRequest" } }, - "external_ips": { + "ephemeral_ip": { + "nullable": true, "description": "Zero or more external IP addresses (either floating or ephemeral), provided to an instance to allow inbound connectivity.", + "type": "string", + "format": "ip" + }, + "firewall_rules": { "type": "array", "items": { - "type": "string", - "format": "ip" + "$ref": "#/components/schemas/VpcFirewallRule" } }, - "firewall_rules": { + "floating_ips": { "type": "array", "items": { - "$ref": "#/components/schemas/VpcFirewallRule" + "type": "string", + "format": "ip" } }, "nics": { @@ -4282,8 +4287,8 @@ "required": [ "dhcp_config", "disks", - "external_ips", "firewall_rules", + "floating_ips", "nics", "properties", "source_nat" diff --git a/package-manifest.toml b/package-manifest.toml index 3bce4aafee..bd60fe9e93 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -425,7 +425,7 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "579592bf474ec4b86805ada60c1b920b3beef5a7" +source.commit = "2fd39b75df696961e5ea190c7d74dd91f4849cd3" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt source.sha256 = "38851c79c85d53e997db748520fb27c82299ce7e58a550e35646a548498f1271" @@ -441,7 +441,7 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "579592bf474ec4b86805ada60c1b920b3beef5a7" +source.commit = "2fd39b75df696961e5ea190c7d74dd91f4849cd3" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt source.sha256 = "8cd94e9a6f6175081ce78f0281085a08a5306cde453d8e21deb28050945b1d88" @@ -456,10 +456,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "579592bf474ec4b86805ada60c1b920b3beef5a7" +source.commit = "2fd39b75df696961e5ea190c7d74dd91f4849cd3" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "82aa1ca1d7701b2221c442d58f912be59798258d574effcb866ffab22753cf38" +source.sha256 = "802636775fa77dc6eec193e65fde87e403f6a11531745d47ef5e7ff13b242890" output.type = "zone" output.intermediate_only = true @@ -476,8 +476,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "2af6adea85c62ac37e451148b84e5eb0ef005f36" -source.sha256 = "dc93b671cce54e83ed55faaa267f81ba9e65abcd6714aa559d68a8783d73b1c1" +source.commit = "45e05b2a90203d84510e0c8e902d9449b09ffd9b" +source.sha256 = "b14e73c8091a004472f9825b9b81b2c685bc5a48801704380a80481499060ad9" output.type = "zone" output.intermediate_only = true @@ -501,8 +501,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "2af6adea85c62ac37e451148b84e5eb0ef005f36" -source.sha256 = "c34b10d47fa3eb9f9f6b3655ea4ed8a726f93399ea177efea79f5c89f2ab5a1e" +source.commit = "45e05b2a90203d84510e0c8e902d9449b09ffd9b" +source.sha256 = "06575bea6173d16f6d206b580956ae2cdc72c65df2eb2f40dac01468ab49e336" output.type = "zone" output.intermediate_only = true @@ -519,8 +519,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "2af6adea85c62ac37e451148b84e5eb0ef005f36" -source.sha256 = "ce7065227c092ee82704f39a966b7441e3ae82d75eedb6eb281bd8b3e5873e32" +source.commit = "45e05b2a90203d84510e0c8e902d9449b09ffd9b" +source.sha256 = "db2a398426fe59bd911eed91a3db7731a7a4d57e31dd357d89828d04b0891e2a" output.type = "zone" output.intermediate_only = true diff --git a/schema/crdb/19.0.0/up01.sql b/schema/crdb/19.0.0/up01.sql new file mode 100644 index 0000000000..6cfa92f4c2 --- /dev/null +++ b/schema/crdb/19.0.0/up01.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.external_ip ADD COLUMN IF NOT EXISTS project_id UUID; diff --git a/schema/crdb/19.0.0/up02.sql b/schema/crdb/19.0.0/up02.sql new file mode 100644 index 0000000000..733c46b0dc --- /dev/null +++ b/schema/crdb/19.0.0/up02.sql @@ -0,0 +1,4 @@ +ALTER TABLE omicron.public.external_ip ADD CONSTRAINT IF NOT EXISTS null_project_id CHECK ( + (kind = 'floating' AND is_service = FALSE AND project_id IS NOT NULL) OR + ((kind != 'floating' OR is_service = TRUE) AND project_id IS NULL) +); diff --git a/schema/crdb/19.0.0/up03.sql b/schema/crdb/19.0.0/up03.sql new file mode 100644 index 0000000000..d3577edc12 --- /dev/null +++ b/schema/crdb/19.0.0/up03.sql @@ -0,0 +1,6 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_floating_ip_by_name on omicron.public.external_ip ( + name +) WHERE + kind = 'floating' AND + time_deleted is NULL AND + project_id is NULL; diff --git a/schema/crdb/19.0.0/up04.sql b/schema/crdb/19.0.0/up04.sql new file mode 100644 index 0000000000..9a40dc99c5 --- /dev/null +++ b/schema/crdb/19.0.0/up04.sql @@ -0,0 +1,7 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_floating_ip_by_name_and_project on omicron.public.external_ip ( + project_id, + name +) WHERE + kind = 'floating' AND + time_deleted is NULL AND + project_id is NOT NULL; diff --git a/schema/crdb/19.0.0/up05.sql b/schema/crdb/19.0.0/up05.sql new file mode 100644 index 0000000000..3e172e3e70 --- /dev/null +++ b/schema/crdb/19.0.0/up05.sql @@ -0,0 +1,19 @@ +CREATE VIEW IF NOT EXISTS omicron.public.floating_ip AS +SELECT + id, + name, + description, + time_created, + time_modified, + time_deleted, + ip_pool_id, + ip_pool_range_id, + is_service, + parent_id, + ip, + project_id +FROM + omicron.public.external_ip +WHERE + omicron.public.external_ip.kind = 'floating' AND + project_id IS NOT NULL; diff --git a/schema/crdb/19.0.0/up06.sql b/schema/crdb/19.0.0/up06.sql new file mode 100644 index 0000000000..30c0b3773a --- /dev/null +++ b/schema/crdb/19.0.0/up06.sql @@ -0,0 +1,3 @@ +ALTER TABLE omicron.public.external_ip ADD CONSTRAINT IF NOT EXISTS null_non_fip_parent_id CHECK ( + (kind != 'floating' AND parent_id is NOT NULL) OR (kind = 'floating') +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index f82829a2d9..0bf365a2f1 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -1662,6 +1662,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.external_ip ( /* The last port in the allowed range, also inclusive. */ last_port INT4 NOT NULL, + /* FK to the `project` table. */ + project_id UUID, + /* The name must be non-NULL iff this is a floating IP. */ CONSTRAINT null_fip_name CHECK ( (kind != 'floating' AND name IS NULL) OR @@ -1674,6 +1677,14 @@ CREATE TABLE IF NOT EXISTS omicron.public.external_ip ( (kind = 'floating' AND description IS NOT NULL) ), + /* Only floating IPs can be attached to a project, and + * they must have a parent project if they are instance FIPs. + */ + CONSTRAINT null_project_id CHECK ( + (kind = 'floating' AND is_service = FALSE AND project_id is NOT NULL) OR + ((kind != 'floating' OR is_service = TRUE) AND project_id IS NULL) + ), + /* * Only nullable if this is a floating IP, which may exist not * attached to any instance or service yet. @@ -1717,6 +1728,43 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_external_ip_by_parent ON omicron.public ) WHERE parent_id IS NOT NULL AND time_deleted IS NULL; +/* Enforce name-uniqueness of floating (service) IPs at fleet level. */ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_floating_ip_by_name on omicron.public.external_ip ( + name +) WHERE + kind = 'floating' AND + time_deleted is NULL AND + project_id is NULL; + +/* Enforce name-uniqueness of floating IPs at project level. */ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_floating_ip_by_name_and_project on omicron.public.external_ip ( + project_id, + name +) WHERE + kind = 'floating' AND + time_deleted is NULL AND + project_id is NOT NULL; + +CREATE VIEW IF NOT EXISTS omicron.public.floating_ip AS +SELECT + id, + name, + description, + time_created, + time_modified, + time_deleted, + ip_pool_id, + ip_pool_range_id, + is_service, + parent_id, + ip, + project_id +FROM + omicron.public.external_ip +WHERE + omicron.public.external_ip.kind = 'floating' AND + project_id IS NOT NULL; + /*******************************************************************/ /* @@ -3014,7 +3062,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '18.0.0', NULL) + ( TRUE, NOW(), NOW(), '19.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index cb411a2546..75958a2f37 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -22,8 +22,8 @@ use mg_admin_client::Client as MgdClient; use omicron_common::address::{Ipv6Subnet, MGD_PORT, MGS_PORT}; use omicron_common::address::{DDMD_PORT, DENDRITE_PORT}; use omicron_common::api::internal::shared::{ - PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, RackNetworkConfigV1, - SwitchLocation, UplinkConfig, + BgpConfig, PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, + RackNetworkConfigV1, SwitchLocation, UplinkConfig, }; use omicron_common::backoff::{ retry_notify, retry_policy_local, BackoffError, ExponentialBackoff, @@ -472,23 +472,37 @@ impl<'a> EarlyNetworkSetup<'a> { )) })?; + let mut config: Option = None; + let mut bgp_peer_configs = HashMap::>::new(); + // Iterate through ports and apply BGP config. for port in &our_ports { - let mut bgp_peer_configs = Vec::new(); for peer in &port.bgp_peers { - let config = rack_network_config - .bgp - .iter() - .find(|x| x.asn == peer.asn) - .ok_or(EarlyNetworkSetupError::BgpConfigurationError( - format!( - "asn {} referenced by peer undefined", - peer.asn - ), - ))?; + if let Some(config) = &config { + if peer.asn != config.asn { + return Err(EarlyNetworkSetupError::BadConfig( + "only one ASN per switch is supported".into(), + )); + } + } else { + config = Some( + rack_network_config + .bgp + .iter() + .find(|x| x.asn == peer.asn) + .ok_or( + EarlyNetworkSetupError::BgpConfigurationError( + format!( + "asn {} referenced by peer undefined", + peer.asn + ), + ), + )? + .clone(), + ); + } let bpc = BgpPeerConfig { - asn: peer.asn, name: format!("{}", peer.addr), host: format!("{}:179", peer.addr), hold_time: peer.hold_time.unwrap_or(6), @@ -497,30 +511,41 @@ impl<'a> EarlyNetworkSetup<'a> { connect_retry: peer.connect_retry.unwrap_or(3), keepalive: peer.keepalive.unwrap_or(2), resolution: BGP_SESSION_RESOLUTION, - originate: config - .originate - .iter() - .map(|x| Prefix4 { length: x.prefix(), value: x.ip() }) - .collect(), + passive: false, }; - bgp_peer_configs.push(bpc); + match bgp_peer_configs.get_mut(&port.port) { + Some(peers) => { + peers.push(bpc); + } + None => { + bgp_peer_configs.insert(port.port.clone(), vec![bpc]); + } + } } + } - if bgp_peer_configs.is_empty() { - continue; + if !bgp_peer_configs.is_empty() { + if let Some(config) = &config { + mgd.inner + .bgp_apply(&ApplyRequest { + asn: config.asn, + peers: bgp_peer_configs, + originate: config + .originate + .iter() + .map(|x| Prefix4 { + length: x.prefix(), + value: x.ip(), + }) + .collect(), + }) + .await + .map_err(|e| { + EarlyNetworkSetupError::BgpConfigurationError(format!( + "BGP peer configuration failed: {e}", + )) + })?; } - - mgd.inner - .bgp_apply(&ApplyRequest { - peer_group: port.port.clone(), - peers: bgp_peer_configs, - }) - .await - .map_err(|e| { - EarlyNetworkSetupError::BgpConfigurationError(format!( - "BGP peer configuration failed: {e}", - )) - })?; } Ok(our_ports) @@ -682,6 +707,65 @@ pub struct EarlyNetworkConfig { pub body: EarlyNetworkConfigBody, } +impl EarlyNetworkConfig { + // Note: This currently only converts between v0 and v1 or deserializes v1 of + // `EarlyNetworkConfig`. + pub fn deserialize_bootstore_config( + log: &Logger, + config: &bootstore::NetworkConfig, + ) -> Result { + // Try to deserialize the latest version of the data structure (v1). If + // that succeeds we are done. + let v1_error = + match serde_json::from_slice::(&config.blob) { + Ok(val) => return Ok(val), + Err(error) => { + // Log this error and continue trying to deserialize older + // versions. + warn!( + log, + "Failed to deserialize EarlyNetworkConfig \ + as v1, trying next as v0: {}", + error, + ); + error + } + }; + + match serde_json::from_slice::(&config.blob) { + Ok(val) => { + // Convert from v0 to v1 + return Ok(EarlyNetworkConfig { + generation: val.generation, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: val.ntp_servers, + rack_network_config: val.rack_network_config.map( + |v0_config| { + RackNetworkConfigV0::to_v1( + val.rack_subnet, + v0_config, + ) + }, + ), + }, + }); + } + Err(error) => { + // Log this error. + warn!( + log, + "Failed to deserialize EarlyNetworkConfig as v0: {}", error, + ); + } + }; + + // Return the v1 error preferentially over the v0 error as it's more + // likely to be useful. + Err(v1_error) + } +} + /// This is the actual configuration of EarlyNetworking. /// /// We nest it below the "header" of `generation` and `schema_version` so that @@ -711,39 +795,6 @@ impl From for bootstore::NetworkConfig { } } -// Note: This currently only converts between v0 and v1 or deserializes v1 of -// `EarlyNetworkConfig`. -impl TryFrom for EarlyNetworkConfig { - type Error = serde_json::Error; - - fn try_from( - value: bootstore::NetworkConfig, - ) -> std::result::Result { - // Try to deserialize the latest version of the data structure (v1). If - // that succeeds we are done. - if let Ok(val) = - serde_json::from_slice::(&value.blob) - { - return Ok(val); - } - - // We don't have the latest version. Try to deserialize v0 and then - // convert it to the latest version. - let v0 = serde_json::from_slice::(&value.blob)?; - - Ok(EarlyNetworkConfig { - generation: v0.generation, - schema_version: 1, - body: EarlyNetworkConfigBody { - ntp_servers: v0.ntp_servers, - rack_network_config: v0.rack_network_config.map(|v0_config| { - RackNetworkConfigV0::to_v1(v0.rack_subnet, v0_config) - }), - }, - }) - } -} - /// Deprecated, use `RackNetworkConfig` instead. Cannot actually deprecate due to /// /// @@ -815,9 +866,13 @@ fn convert_fec(fec: &PortFec) -> dpd_client::types::PortFec { mod tests { use super::*; use omicron_common::api::internal::shared::RouteConfig; + use omicron_test_utils::dev::test_setup_log; #[test] fn serialized_early_network_config_v0_to_v1_conversion() { + let logctx = test_setup_log( + "serialized_early_network_config_v0_to_v1_conversion", + ); let v0 = EarlyNetworkConfigV0 { generation: 1, rack_subnet: Ipv6Addr::UNSPECIFIED, @@ -841,7 +896,11 @@ mod tests { let bootstore_conf = bootstore::NetworkConfig { generation: 1, blob: v0_serialized }; - let v1 = EarlyNetworkConfig::try_from(bootstore_conf).unwrap(); + let v1 = EarlyNetworkConfig::deserialize_bootstore_config( + &logctx.log, + &bootstore_conf, + ) + .unwrap(); let v0_rack_network_config = v0.rack_network_config.unwrap(); let uplink = v0_rack_network_config.uplinks[0].clone(); let expected = EarlyNetworkConfig { @@ -872,5 +931,7 @@ mod tests { }; assert_eq!(expected, v1); + + logctx.cleanup_successful(); } } diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 9c3a079dac..2dcb35b77e 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -659,7 +659,10 @@ async fn read_network_bootstore_config_cache( })?; let config = match config { - Some(config) => EarlyNetworkConfig::try_from(config).map_err(|e| { + Some(config) => EarlyNetworkConfig::deserialize_bootstore_config( + &rqctx.log, &config, + ) + .map_err(|e| { HttpError::for_internal_error(format!( "deserialize early network config: {e}" )) diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index c37f0ffde6..a811678a48 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -208,7 +208,8 @@ struct InstanceInner { // Guest NIC and OPTE port information requested_nics: Vec, source_nat: SourceNatConfig, - external_ips: Vec, + ephemeral_ip: Option, + floating_ips: Vec, firewall_rules: Vec, dhcp_config: DhcpCfg, @@ -669,7 +670,8 @@ impl Instance { port_manager, requested_nics: hardware.nics, source_nat: hardware.source_nat, - external_ips: hardware.external_ips, + ephemeral_ip: hardware.ephemeral_ip, + floating_ips: hardware.floating_ips, firewall_rules: hardware.firewall_rules, dhcp_config, requested_disks: hardware.disks, @@ -882,15 +884,20 @@ impl Instance { // Create OPTE ports for the instance let mut opte_ports = Vec::with_capacity(inner.requested_nics.len()); for nic in inner.requested_nics.iter() { - let (snat, external_ips) = if nic.primary { - (Some(inner.source_nat), &inner.external_ips[..]) + let (snat, ephemeral_ip, floating_ips) = if nic.primary { + ( + Some(inner.source_nat), + inner.ephemeral_ip, + &inner.floating_ips[..], + ) } else { - (None, &[][..]) + (None, None, &[][..]) }; let port = inner.port_manager.create_port( nic, snat, - external_ips, + ephemeral_ip, + floating_ips, &inner.firewall_rules, inner.dhcp_config.clone(), )?; diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 6be2ceabbd..a7d91e2b93 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -70,7 +70,8 @@ pub struct InstanceHardware { pub source_nat: SourceNatConfig, /// Zero or more external IP addresses (either floating or ephemeral), /// provided to an instance to allow inbound connectivity. - pub external_ips: Vec, + pub ephemeral_ip: Option, + pub floating_ips: Vec, pub firewall_rules: Vec, pub dhcp_config: DhcpConfig, // TODO: replace `propolis_client::*` with locally-modeled request type diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index dc309e8423..651d2638e0 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -845,12 +845,7 @@ impl ServiceManager { None, omicron_zones_config.clone(), |z: &OmicronZoneConfig| { - matches!( - z.zone_type, - OmicronZoneType::InternalDns { .. } - | OmicronZoneType::BoundaryNtp { .. } - | OmicronZoneType::InternalNtp { .. } - ) + matches!(z.zone_type, OmicronZoneType::InternalDns { .. }) }, ) .await?; @@ -859,8 +854,6 @@ impl ServiceManager { // synchronization, which is a pre-requisite for the other services. We // keep `OmicronZoneType::InternalDns` because // `ensure_all_omicron_zones` is additive. - // TODO This looks like a duplicate of the block above -- why do we do - // this? let all_zones_request = self .ensure_all_omicron_zones( &mut existing_zones, @@ -1167,7 +1160,7 @@ impl ServiceManager { .collect(); let external_ip; - let (zone_type_str, nic, snat, external_ips) = match &zone_args + let (zone_type_str, nic, snat, floating_ips) = match &zone_args .omicron_type() { Some( @@ -1207,16 +1200,18 @@ impl ServiceManager { // config allows outbound access which is enough for // Boundary NTP which needs to come up before Nexus. let port = port_manager - .create_port(nic, snat, external_ips, &[], DhcpCfg::default()) + .create_port(nic, snat, None, floating_ips, &[], DhcpCfg::default()) .map_err(|err| Error::ServicePortCreation { service: zone_type_str.clone(), err: Box::new(err), })?; // We also need to update the switch with the NAT mappings + // XXX: need to revisit iff. any services get more than one + // address. let (target_ip, first_port, last_port) = match snat { Some(s) => (s.ip, s.first_port, s.last_port), - None => (external_ips[0], 0, u16::MAX), + None => (floating_ips[0], 0, u16::MAX), }; for dpd_client in &dpd_clients { diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 90e9706198..57aea61ae9 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -445,8 +445,11 @@ impl SledAgent { })?; let early_network_config = - EarlyNetworkConfig::try_from(serialized_config) - .map_err(|err| BackoffError::transient(err.to_string()))?; + EarlyNetworkConfig::deserialize_bootstore_config( + &log, + &serialized_config, + ) + .map_err(|err| BackoffError::transient(err.to_string()))?; Ok(early_network_config.body.rack_network_config) }; diff --git a/sled-agent/tests/data/early_network_blobs.txt b/sled-agent/tests/data/early_network_blobs.txt new file mode 100644 index 0000000000..c968d4010b --- /dev/null +++ b/sled-agent/tests/data/early_network_blobs.txt @@ -0,0 +1,2 @@ +2023-11-30 mupdate failing blob,{"generation":15,"schema_version":1,"body":{"ntp_servers":[],"rack_network_config":{"rack_subnet":"fd00:1122:3344:100::/56","infra_ip_first":"0.0.0.0","infra_ip_last":"0.0.0.0","ports":[{"routes":[],"addresses":[],"switch":"switch1","port":"qsfp0","uplink_port_speed":"speed100_g","uplink_port_fec":"none","bgp_peers":[]},{"routes":[],"addresses":["172.20.15.53/29"],"switch":"switch1","port":"qsfp18","uplink_port_speed":"speed100_g","uplink_port_fec":"rs","bgp_peers":[{"asn":65002,"port":"qsfp18","addr":"172.20.15.51","hold_time":6,"idle_hold_time":6,"delay_open":0,"connect_retry":3,"keepalive":2}]},{"routes":[],"addresses":["172.20.15.45/29"],"switch":"switch0","port":"qsfp18","uplink_port_speed":"speed100_g","uplink_port_fec":"rs","bgp_peers":[{"asn":65002,"port":"qsfp18","addr":"172.20.15.43","hold_time":6,"idle_hold_time":6,"delay_open":0,"connect_retry":3,"keepalive":2}]},{"routes":[],"addresses":[],"switch":"switch0","port":"qsfp0","uplink_port_speed":"speed100_g","uplink_port_fec":"none","bgp_peers":[]}],"bgp":[{"asn":65002,"originate":["172.20.26.0/24"]},{"asn":65002,"originate":["172.20.26.0/24"]}]}}} +2023-12-06 config,{"generation":20,"schema_version":1,"body":{"ntp_servers":["ntp.example.com"],"rack_network_config":{"rack_subnet":"ff01::/32","infra_ip_first":"127.0.0.1","infra_ip_last":"127.1.0.1","ports":[{"routes":[{"destination":"10.1.9.32/16","nexthop":"10.1.9.32"}],"addresses":["2001:db8::/96"],"switch":"switch0","port":"foo","uplink_port_speed":"speed200_g","uplink_port_fec":"firecode","bgp_peers":[{"asn":65000,"port":"bar","addr":"1.2.3.4","hold_time":20,"idle_hold_time":50,"delay_open":null,"connect_retry":30,"keepalive":10}],"autoneg":true}],"bgp":[{"asn":20000,"originate":["192.168.0.0/24"]}]}}} diff --git a/sled-agent/tests/integration_tests/early_network.rs b/sled-agent/tests/integration_tests/early_network.rs new file mode 100644 index 0000000000..c3a4a53ebf --- /dev/null +++ b/sled-agent/tests/integration_tests/early_network.rs @@ -0,0 +1,154 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tests that EarlyNetworkConfig deserializes across versions. + +use std::net::Ipv4Addr; + +use bootstore::schemes::v0 as bootstore; +use omicron_common::api::{ + external::SwitchLocation, + internal::shared::{ + BgpConfig, BgpPeerConfig, PortConfigV1, PortFec, PortSpeed, + RackNetworkConfig, RouteConfig, + }, +}; +use omicron_sled_agent::bootstrap::early_networking::{ + EarlyNetworkConfig, EarlyNetworkConfigBody, +}; +use omicron_test_utils::dev::test_setup_log; + +const BLOB_PATH: &str = "tests/data/early_network_blobs.txt"; + +/// Test that previous and current versions of `EarlyNetworkConfig` blobs +/// deserialize correctly. +#[test] +fn early_network_blobs_deserialize() { + let logctx = test_setup_log("early_network_blobs_deserialize"); + + let (current_desc, current_config) = current_config_example(); + assert!( + !current_desc.contains(',') && !current_desc.contains('\n'), + "current_desc must not contain commas or newlines" + ); + + // Read old blobs as newline-delimited JSON. + let mut known_blobs = std::fs::read_to_string(BLOB_PATH) + .expect("error reading early_network_blobs.txt"); + let mut current_blob_is_known = false; + for (blob_idx, line) in known_blobs.lines().enumerate() { + let blob_lineno = blob_idx + 1; + let (blob_desc, blob_json) = + line.split_once(',').unwrap_or_else(|| { + panic!( + "error parsing early_network_blobs.txt \ + line {blob_lineno}: missing comma", + ); + }); + + // Attempt to deserialize this blob. + let config = serde_json::from_str::(blob_json) + .unwrap_or_else(|error| { + panic!( + "error deserializing early_network_blobs.txt \ + \"{blob_desc}\" (line {blob_lineno}): {error}", + ); + }); + + // Does this config match the current config? + if blob_desc == current_desc { + assert_eq!( + config, current_config, + "early_network_blobs.txt line {}: {} does not match current config", + blob_lineno, blob_desc + ); + current_blob_is_known = true; + } + + // Now attempt to put this blob into a bootstore config, and deserialize that. + let network_config = bootstore::NetworkConfig { + generation: config.generation, + blob: blob_json.to_owned().into(), + }; + let config2 = EarlyNetworkConfig::deserialize_bootstore_config( + &logctx.log, + &network_config, + ).unwrap_or_else(|error| { + panic!( + "error deserializing early_network_blobs.txt \ + \"{blob_desc}\" (line {blob_lineno}) as bootstore config: {error}", + ); + }); + + assert_eq!( + config, config2, + "early_network_blobs.txt line {}: {} does not match deserialization \ + as bootstore config", + blob_lineno, blob_desc + ); + } + + // If the current blob was not covered, add it to the list of known blobs. + if !current_blob_is_known { + let current_blob_json = serde_json::to_string(¤t_config).unwrap(); + let current_blob = format!("{},{}", current_desc, current_blob_json); + known_blobs.push_str(¤t_blob); + known_blobs.push('\n'); + } + + expectorate::assert_contents(BLOB_PATH, &known_blobs); + + logctx.cleanup_successful(); +} + +/// Returns a current version of the EarlyNetworkConfig blob, along with a +/// short description of the current version. The values can be arbitrary, but +/// this should be a nontrivial blob where no vectors are empty. +/// +/// The goal is that if the definition of `EarlyNetworkConfig` changes in the +/// future, older blobs can still be deserialized correctly. +fn current_config_example() -> (&'static str, EarlyNetworkConfig) { + // NOTE: the description must not contain commas or newlines. + let description = "2023-12-06 config"; + let config = EarlyNetworkConfig { + generation: 20, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: vec!["ntp.example.com".to_owned()], + rack_network_config: Some(RackNetworkConfig { + rack_subnet: "ff01::0/32".parse().unwrap(), + infra_ip_first: Ipv4Addr::new(127, 0, 0, 1), + infra_ip_last: Ipv4Addr::new(127, 1, 0, 1), + ports: vec![PortConfigV1 { + routes: vec![RouteConfig { + destination: "10.1.9.32/16".parse().unwrap(), + nexthop: "10.1.9.32".parse().unwrap(), + }], + addresses: vec!["2001:db8::/96".parse().unwrap()], + switch: SwitchLocation::Switch0, + port: "foo".to_owned(), + uplink_port_speed: PortSpeed::Speed200G, + uplink_port_fec: PortFec::Firecode, + bgp_peers: vec![BgpPeerConfig { + asn: 65000, + port: "bar".to_owned(), + addr: Ipv4Addr::new(1, 2, 3, 4), + hold_time: Some(20), + idle_hold_time: Some(50), + delay_open: None, + connect_retry: Some(30), + keepalive: Some(10), + }], + autoneg: true, + }], + bgp: vec![BgpConfig { + asn: 20000, + originate: vec!["192.168.0.0/24".parse().unwrap()], + }], + }), + }, + }; + + (description, config) +} diff --git a/sled-agent/tests/integration_tests/mod.rs b/sled-agent/tests/integration_tests/mod.rs index 1bf43dc00c..13e38077ea 100644 --- a/sled-agent/tests/integration_tests/mod.rs +++ b/sled-agent/tests/integration_tests/mod.rs @@ -3,3 +3,4 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. mod commands; +mod early_network; diff --git a/test-utils/src/certificates.rs b/test-utils/src/certificates.rs index ab84f30b15..54da013e0c 100644 --- a/test-utils/src/certificates.rs +++ b/test-utils/src/certificates.rs @@ -79,10 +79,10 @@ impl CertificateChain { fn tls_cert_to_pem(certs: &Vec) -> String { let mut serialized_certs = String::new(); for cert in certs { - let encoded_cert = pem::encode(&pem::Pem { - tag: "CERTIFICATE".to_string(), - contents: cert.0.clone(), - }); + let encoded_cert = pem::encode(&pem::Pem::new( + "CERTIFICATE".to_string(), + cert.0.clone(), + )); serialized_certs.push_str(&encoded_cert); } diff --git a/tools/ci_download_softnpu_machinery b/tools/ci_download_softnpu_machinery index 3efb030063..e147238673 100755 --- a/tools/ci_download_softnpu_machinery +++ b/tools/ci_download_softnpu_machinery @@ -15,7 +15,7 @@ OUT_DIR="out/npuzone" # Pinned commit for softnpu ASIC simulator SOFTNPU_REPO="softnpu" -SOFTNPU_COMMIT="dec63e67156fe6e958991bbfa090629868115ab5" +SOFTNPU_COMMIT="dbab082dfa89da5db5ca2325c257089d2f130092" # This is the softnpu ASIC simulator echo "fetching npuzone" diff --git a/tools/console_version b/tools/console_version index 811620e9e7..725bda0ee9 100644 --- a/tools/console_version +++ b/tools/console_version @@ -1,2 +1,2 @@ -COMMIT="ae8218df707360a902133f4a96b48a3b5a62a09e" -SHA2="ae35b991d3ff835a59b59126298790cb7431a282b25ba4add4e7fb6ea6b98989" +COMMIT="1802c2859f40712017ab89e72740e39bfd59320b" +SHA2="34768a895f187a6ed263c0050c42084f3907c331b547362871c2ce330e9d08d1" diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index 884d356222..7721fb1c0f 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -63,8 +63,8 @@ function ensure_softnpu_zone { --omicron-zone \ --ports sc0_0,tfportrear0_0 \ --ports sc0_1,tfportqsfp0_0 \ - --sidecar-lite-commit f0585a29fb0285f7a1220c1118856b0e5c1f75c5 \ - --softnpu-commit dec63e67156fe6e958991bbfa090629868115ab5 + --sidecar-lite-commit 45ed98fea5824feb4d42f45bbf218e597dc9fc58 \ + --softnpu-commit dbab082dfa89da5db5ca2325c257089d2f130092 } "$SOURCE_DIR"/scrimlet/softnpu-init.sh success "softnpu zone exists" diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index c2dda4dbd0..c2afe5ca87 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="2af6adea85c62ac37e451148b84e5eb0ef005f36" +COMMIT="45e05b2a90203d84510e0c8e902d9449b09ffd9b" SHA2="07d115bfa8498a8015ca2a8447efeeac32e24aeb25baf3d5e2313216e11293c0" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index 77ee198fc5..2b4f0e7555 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="dc93b671cce54e83ed55faaa267f81ba9e65abcd6714aa559d68a8783d73b1c1" -CIDL_SHA256_LINUX_DPD="b13b391a085ba6bf16fdd99774f64c9d53cd7220ad518d5839c8558fb925c40c" -CIDL_SHA256_LINUX_SWADM="6bfa4e367eb2b0be89f1588ac458026a186314597a4feb9fee6cea60101c7ebe" +CIDL_SHA256_ILLUMOS="b14e73c8091a004472f9825b9b81b2c685bc5a48801704380a80481499060ad9" +CIDL_SHA256_LINUX_DPD="a0d92b5007826b119c68fdaef753e33b125740ec7b3e771bfa6b3aa8d9fcb8cc" +CIDL_SHA256_LINUX_SWADM="13387460db5b57e6ffad6c0b8877af32cc6d53fecc4a1a0910143c0446d39a38" diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index f60ea76380..37c099d7f5 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="579592bf474ec4b86805ada60c1b920b3beef5a7" +COMMIT="2fd39b75df696961e5ea190c7d74dd91f4849cd3" SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 649db53f6e..329c05fc42 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="579592bf474ec4b86805ada60c1b920b3beef5a7" -SHA2="6c1fab8d5028b52a161d8bf02aae47844699cdc5f7b28e1ac519fc4ec1ab3971" +COMMIT="2fd39b75df696961e5ea190c7d74dd91f4849cd3" +SHA2="931efa310d972b1f8afba2308751fc6a2035afbaebba77b3a40a8358c123ba3c" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 08b04d6b67..1d3cf98f94 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="82aa1ca1d7701b2221c442d58f912be59798258d574effcb866ffab22753cf38" -MGD_LINUX_SHA256="81231b30872fa1c581aa22c101f32d11f33f335758ac1fd2653436fbc7aab93f" \ No newline at end of file +CIDL_SHA256="802636775fa77dc6eec193e65fde87e403f6a11531745d47ef5e7ff13b242890" +MGD_LINUX_SHA256="1bcadfd700902e3640843e0bb53d3defdbcd8d86c3279efa0953ae8d6437e2b0" \ No newline at end of file diff --git a/tools/opte_version b/tools/opte_version index 0a79a6aba9..fa0ef8d768 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.25.183 +0.27.199 diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index a86ea35cc3..7faaa08a28 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -181,14 +181,19 @@ impl UpdateTracker { } /// Starts a fake update that doesn't perform any steps, but simply waits - /// for a watch receiver to resolve. + /// for a receiver to resolve. + /// + /// The inner sender will resolve once the update is completed. #[doc(hidden)] pub async fn start_fake_update( &self, sps: BTreeSet, - watch_receiver: watch::Receiver<()>, + fake_step_receiver: oneshot::Receiver>, ) -> Result<(), Vec> { - let imp = FakeUpdateDriver { watch_receiver, log: self.log.clone() }; + let imp = FakeUpdateDriver { + fake_step_receiver: Some(fake_step_receiver), + log: self.log.clone(), + }; self.start_impl(sps, Some(imp)).await } @@ -515,7 +520,7 @@ impl<'tr> SpawnUpdateDriver for RealSpawnUpdateDriver<'tr> { /// waits for a [`watch::Receiver`] to resolve. #[derive(Debug)] struct FakeUpdateDriver { - watch_receiver: watch::Receiver<()>, + fake_step_receiver: Option>>, log: Logger, } @@ -539,22 +544,24 @@ impl SpawnUpdateDriver for FakeUpdateDriver { let engine = UpdateEngine::new(&log, sender); let abort_handle = engine.abort_handle(); - let mut watch_receiver = self.watch_receiver.clone(); + let fake_step_receiver = self + .fake_step_receiver + .take() + .expect("fake step receiver is only taken once"); let task = tokio::spawn(async move { // The step component and ID have been chosen arbitrarily here -- // they aren't important. - engine + let final_sender_handle = engine .new_step( UpdateComponent::Host, UpdateStepId::RunningInstallinator, "Fake step that waits for receiver to resolve", move |_cx| async move { - // This will resolve as soon as the watch sender - // (typically a test) sends a value over the watch - // channel. - _ = watch_receiver.changed().await; - StepSuccess::new(()).into() + // This will resolve as soon as the sender (typically a + // test) sends a value over the channel. + let ret = fake_step_receiver.await; + StepSuccess::new(ret).into() }, ) .register(); @@ -566,16 +573,36 @@ impl SpawnUpdateDriver for FakeUpdateDriver { } }); - match engine.execute().await { - Ok(_cx) => (), - Err(err) => { - error!(log, "update failed"; "err" => %err); - } - } + let engine_res = engine.execute().await; // Wait for all events to be received and written to the event // buffer. event_receiving_task.await.expect("event receiving task panicked"); + + // Finally, notify the receiving end of the inner sender: this + // indicates that the update is done. + match engine_res { + Ok(cx) => { + info!(log, "fake update completed successfully"); + let final_sender = + final_sender_handle.into_value(cx.token()).await; + match final_sender { + Ok(sender) => { + if let Err(_) = sender.send(()) { + warn!(log, "failed to send final value"); + } + } + Err(error) => { + // This occurs if the fake_step_receiver's sender + // side was closed. Nothing to do here but warn. + warn!(log, "failed to get final sender: {}", error); + } + } + } + Err(error) => { + error!(log, "fake update failed: {}", error); + } + } }); SpUpdateData { task, abort_handle, event_buffer } diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs index 52bf1d1283..611d81c7f5 100644 --- a/wicketd/tests/integration_tests/updates.rs +++ b/wicketd/tests/integration_tests/updates.rs @@ -17,7 +17,7 @@ use omicron_common::{ api::internal::nexus::KnownArtifactKind, update::{ArtifactHashId, ArtifactKind}, }; -use tokio::sync::watch; +use tokio::sync::oneshot; use update_engine::NestedError; use uuid::Uuid; use wicket::OutputKind; @@ -436,7 +436,7 @@ async fn test_update_races() { }; let sps: BTreeSet<_> = vec![sp].into_iter().collect(); - let (sender, receiver) = watch::channel(()); + let (sender, receiver) = oneshot::channel(); wicketd_testctx .server .update_tracker @@ -455,7 +455,7 @@ async fn test_update_races() { // Also try starting another fake update, which should fail -- we don't let updates be started // if there's current update state. { - let (_, receiver) = watch::channel(()); + let (_, receiver) = oneshot::channel(); let err = wicketd_testctx .server .update_tracker @@ -470,9 +470,10 @@ async fn test_update_races() { } // Unblock the update, letting it run to completion. - sender.send(()).expect("receiver kept open by update engine"); + let (final_sender, final_receiver) = oneshot::channel(); + sender.send(final_sender).expect("receiver kept open by update engine"); + final_receiver.await.expect("update engine completed successfully"); - // Ensure that the event buffer indicates completion. let event_buffer = wicketd_testctx .wicketd_client .get_update_sp(&SpType::Sled, 0) diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index fe7c3bdc81..ce65ddf062 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -15,6 +15,7 @@ publish = false ### BEGIN HAKARI SECTION [dependencies] anyhow = { version = "1.0.75", features = ["backtrace"] } +base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } @@ -33,9 +34,12 @@ crossbeam-epoch = { version = "0.9.15" } crossbeam-utils = { version = "0.8.16" } crossterm = { version = "0.27.0", features = ["event-stream", "serde"] } crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } +der = { version = "0.7.8", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] } diesel = { version = "2.1.4", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0" } +elliptic-curve = { version = "0.13.8", features = ["ecdh", "hazmat", "pem", "std"] } +ff = { version = "0.13.0", default-features = false, features = ["alloc"] } flate2 = { version = "1.0.28" } futures = { version = "0.3.29" } futures-channel = { version = "0.3.29", features = ["sink"] } @@ -47,8 +51,10 @@ futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } +group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown = { version = "0.13.2" } hex = { version = "0.4.3", features = ["serde"] } +hmac = { version = "0.12.1", default-features = false, features = ["reset"] } hyper = { version = "0.14.27", features = ["full"] } indexmap = { version = "2.1.0", features = ["serde"] } inout = { version = "0.1.3", default-features = false, features = ["std"] } @@ -65,6 +71,7 @@ num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } openapiv3 = { version = "2.0.0-rc.1", default-features = false, features = ["skip_serializing_defaults"] } +pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] } petgraph = { version = "0.6.4", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } @@ -76,13 +83,12 @@ regex = { version = "1.10.2" } regex-automata = { version = "0.4.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8.2" } reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] } -ring = { version = "0.16.20", features = ["std"] } +ring = { version = "0.17.7", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.20", features = ["serde"] } serde = { version = "1.0.192", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.108", features = ["raw_value"] } sha2 = { version = "0.10.8", features = ["oid"] } -signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } snafu = { version = "0.7.5", features = ["futures"] } @@ -110,6 +116,7 @@ zip = { version = "0.6.6", default-features = false, features = ["bzip2", "defla [build-dependencies] anyhow = { version = "1.0.75", features = ["backtrace"] } +base16ct = { version = "0.2.0", default-features = false, features = ["alloc"] } bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } @@ -128,9 +135,12 @@ crossbeam-epoch = { version = "0.9.15" } crossbeam-utils = { version = "0.8.16" } crossterm = { version = "0.27.0", features = ["event-stream", "serde"] } crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } +der = { version = "0.7.8", default-features = false, features = ["derive", "flagset", "oid", "pem", "std"] } diesel = { version = "2.1.4", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } either = { version = "1.9.0" } +elliptic-curve = { version = "0.13.8", features = ["ecdh", "hazmat", "pem", "std"] } +ff = { version = "0.13.0", default-features = false, features = ["alloc"] } flate2 = { version = "1.0.28" } futures = { version = "0.3.29" } futures-channel = { version = "0.3.29", features = ["sink"] } @@ -142,8 +152,10 @@ futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } +group = { version = "0.13.0", default-features = false, features = ["alloc"] } hashbrown = { version = "0.13.2" } hex = { version = "0.4.3", features = ["serde"] } +hmac = { version = "0.12.1", default-features = false, features = ["reset"] } hyper = { version = "0.14.27", features = ["full"] } indexmap = { version = "2.1.0", features = ["serde"] } inout = { version = "0.1.3", default-features = false, features = ["std"] } @@ -160,6 +172,7 @@ num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } openapiv3 = { version = "2.0.0-rc.1", default-features = false, features = ["skip_serializing_defaults"] } +pem-rfc7468 = { version = "0.7.0", default-features = false, features = ["std"] } petgraph = { version = "0.6.4", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } @@ -171,13 +184,12 @@ regex = { version = "1.10.2" } regex-automata = { version = "0.4.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8.2" } reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] } -ring = { version = "0.16.20", features = ["std"] } +ring = { version = "0.17.7", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.20", features = ["serde"] } serde = { version = "1.0.192", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.108", features = ["raw_value"] } sha2 = { version = "0.10.8", features = ["oid"] } -signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug", "release_max_level_trace"] } snafu = { version = "0.7.5", features = ["futures"] } @@ -208,14 +220,14 @@ zip = { version = "0.6.6", default-features = false, features = ["bzip2", "defla bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.9", features = ["net", "os-ext"] } -once_cell = { version = "1.18.0", features = ["unstable"] } +once_cell = { version = "1.19.0", features = ["unstable"] } rustix = { version = "0.38.25", features = ["fs", "termios"] } [target.x86_64-unknown-linux-gnu.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.9", features = ["net", "os-ext"] } -once_cell = { version = "1.18.0", features = ["unstable"] } +once_cell = { version = "1.19.0", features = ["unstable"] } rustix = { version = "0.38.25", features = ["fs", "termios"] } [target.x86_64-apple-darwin.dependencies] @@ -223,7 +235,7 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f errno = { version = "0.3.2", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.9", features = ["net", "os-ext"] } -once_cell = { version = "1.18.0", features = ["unstable"] } +once_cell = { version = "1.19.0", features = ["unstable"] } rustix = { version = "0.38.25", features = ["fs", "termios"] } [target.x86_64-apple-darwin.build-dependencies] @@ -231,7 +243,7 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f errno = { version = "0.3.2", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.9", features = ["net", "os-ext"] } -once_cell = { version = "1.18.0", features = ["unstable"] } +once_cell = { version = "1.19.0", features = ["unstable"] } rustix = { version = "0.38.25", features = ["fs", "termios"] } [target.aarch64-apple-darwin.dependencies] @@ -239,7 +251,7 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f errno = { version = "0.3.2", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.9", features = ["net", "os-ext"] } -once_cell = { version = "1.18.0", features = ["unstable"] } +once_cell = { version = "1.19.0", features = ["unstable"] } rustix = { version = "0.38.25", features = ["fs", "termios"] } [target.aarch64-apple-darwin.build-dependencies] @@ -247,7 +259,7 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f errno = { version = "0.3.2", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.9", features = ["net", "os-ext"] } -once_cell = { version = "1.18.0", features = ["unstable"] } +once_cell = { version = "1.19.0", features = ["unstable"] } rustix = { version = "0.38.25", features = ["fs", "termios"] } [target.x86_64-unknown-illumos.dependencies] @@ -255,7 +267,7 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f errno = { version = "0.3.2", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.9", features = ["net", "os-ext"] } -once_cell = { version = "1.18.0", features = ["unstable"] } +once_cell = { version = "1.19.0", features = ["unstable"] } rustix = { version = "0.38.25", features = ["fs", "termios"] } toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } @@ -265,7 +277,7 @@ bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-f errno = { version = "0.3.2", default-features = false, features = ["std"] } hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.9", features = ["net", "os-ext"] } -once_cell = { version = "1.18.0", features = ["unstable"] } +once_cell = { version = "1.19.0", features = ["unstable"] } rustix = { version = "0.38.25", features = ["fs", "termios"] } toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] } toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] }