From 92aed1a25a57d8483fc212ba245d871514aeadee Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Fri, 1 Dec 2023 05:17:22 +0000 Subject: [PATCH 01/16] Update taiki-e/install-action digest to 21526ba (#4593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [taiki-e/install-action](https://togithub.com/taiki-e/install-action) | action | digest | [`6b385b7` -> `21526ba`](https://togithub.com/taiki-e/install-action/compare/6b385b7...21526ba) | --- ### Configuration 📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone America/Los_Angeles. 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://togithub.com/renovatebot/renovate). Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com> --- .github/workflows/hakari.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index afc56f40ca..b5a7504066 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -24,7 +24,7 @@ jobs: with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@6b385b7509c65e9d1b7d6b72244f7e275a7f5cef # v2 + uses: taiki-e/install-action@21526ba3bb38834e625c185ae4f2f942f1fb8f27 # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date From ac94fc0e8f53c44f7655994068833d361b81c004 Mon Sep 17 00:00:00 2001 From: Alan Hanson Date: Fri, 1 Dec 2023 14:09:41 -0800 Subject: [PATCH 02/16] Update Cargo.toml to match package-manifest for crucible (#4597) Co-authored-by: Alan Hanson --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 12 ++++++------ sled-agent/src/sim/http_entrypoints_pantry.rs | 7 ++++--- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b730cbda97..06c1c2b5b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -458,7 +458,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" +source = "git+https://github.com/oxidecomputer/propolis?rev=3e1d129151c3621d28ead5c6e5760693ba6e7fec#3e1d129151c3621d28ead5c6e5760693ba6e7fec" dependencies = [ "bhyve_api_sys", "libc", @@ -468,7 +468,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" +source = "git+https://github.com/oxidecomputer/propolis?rev=3e1d129151c3621d28ead5c6e5760693ba6e7fec#3e1d129151c3621d28ead5c6e5760693ba6e7fec" dependencies = [ "libc", "strum", @@ -1281,7 +1281,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=51a3121c8318fc7ac97d74f917ce1d37962e785f#51a3121c8318fc7ac97d74f917ce1d37962e785f" +source = "git+https://github.com/oxidecomputer/crucible?rev=945f040d259ca8013d3fb26f510453da7cd7b1a6#945f040d259ca8013d3fb26f510453da7cd7b1a6" dependencies = [ "anyhow", "chrono", @@ -1297,7 +1297,7 @@ dependencies = [ [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=51a3121c8318fc7ac97d74f917ce1d37962e785f#51a3121c8318fc7ac97d74f917ce1d37962e785f" +source = "git+https://github.com/oxidecomputer/crucible?rev=945f040d259ca8013d3fb26f510453da7cd7b1a6#945f040d259ca8013d3fb26f510453da7cd7b1a6" dependencies = [ "anyhow", "chrono", @@ -1314,7 +1314,7 @@ dependencies = [ [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=51a3121c8318fc7ac97d74f917ce1d37962e785f#51a3121c8318fc7ac97d74f917ce1d37962e785f" +source = "git+https://github.com/oxidecomputer/crucible?rev=945f040d259ca8013d3fb26f510453da7cd7b1a6#945f040d259ca8013d3fb26f510453da7cd7b1a6" dependencies = [ "crucible-workspace-hack", "libc", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" +source = "git+https://github.com/oxidecomputer/propolis?rev=3e1d129151c3621d28ead5c6e5760693ba6e7fec#3e1d129151c3621d28ead5c6e5760693ba6e7fec" dependencies = [ "async-trait", "base64 0.21.5", @@ -6133,7 +6133,7 @@ dependencies = [ [[package]] name = "propolis-mock-server" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" +source = "git+https://github.com/oxidecomputer/propolis?rev=3e1d129151c3621d28ead5c6e5760693ba6e7fec#3e1d129151c3621d28ead5c6e5760693ba6e7fec" dependencies = [ "anyhow", "atty", @@ -6163,7 +6163,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=54398875a2125227d13827d4236dce943c019b1c#54398875a2125227d13827d4236dce943c019b1c" +source = "git+https://github.com/oxidecomputer/propolis?rev=3e1d129151c3621d28ead5c6e5760693ba6e7fec#3e1d129151c3621d28ead5c6e5760693ba6e7fec" dependencies = [ "schemars", "serde", diff --git a/Cargo.toml b/Cargo.toml index 694cd2c8dc..6d47909399 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,9 +171,9 @@ cookie = "0.18" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "51a3121c8318fc7ac97d74f917ce1d37962e785f" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "51a3121c8318fc7ac97d74f917ce1d37962e785f" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "51a3121c8318fc7ac97d74f917ce1d37962e785f" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "945f040d259ca8013d3fb26f510453da7cd7b1a6" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "945f040d259ca8013d3fb26f510453da7cd7b1a6" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "945f040d259ca8013d3fb26f510453da7cd7b1a6" } curve25519-dalek = "4" datatest-stable = "0.2.3" display-error-chain = "0.2.0" @@ -292,9 +292,9 @@ pretty-hex = "0.4.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" } -propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "3e1d129151c3621d28ead5c6e5760693ba6e7fec" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "3e1d129151c3621d28ead5c6e5760693ba6e7fec" } +propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "3e1d129151c3621d28ead5c6e5760693ba6e7fec" } proptest = "1.4.0" quote = "1.0" rand = "0.8.5" diff --git a/sled-agent/src/sim/http_entrypoints_pantry.rs b/sled-agent/src/sim/http_entrypoints_pantry.rs index 64b26a83a4..8430dc0731 100644 --- a/sled-agent/src/sim/http_entrypoints_pantry.rs +++ b/sled-agent/src/sim/http_entrypoints_pantry.rs @@ -96,7 +96,7 @@ struct JobPollResponse { /// Poll to see if a Pantry background job is done #[endpoint { method = GET, - path = "/crucible/pantry/0/job/{id}/is_finished", + path = "/crucible/pantry/0/job/{id}/is-finished", }] async fn is_job_finished( rc: RequestContext>, @@ -139,6 +139,7 @@ async fn job_result_ok( } #[derive(Debug, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] pub enum ExpectedDigest { Sha256(String), } @@ -157,7 +158,7 @@ struct ImportFromUrlResponse { /// Import data from a URL into a volume #[endpoint { method = POST, - path = "/crucible/pantry/0/volume/{id}/import_from_url", + path = "/crucible/pantry/0/volume/{id}/import-from-url", }] async fn import_from_url( rc: RequestContext>, @@ -213,7 +214,7 @@ struct BulkWriteRequest { /// Bulk write data into a volume at a specified offset #[endpoint { method = POST, - path = "/crucible/pantry/0/volume/{id}/bulk_write", + path = "/crucible/pantry/0/volume/{id}/bulk-write", }] async fn bulk_write( rc: RequestContext>, From 8cf3e2a0f9a312e806a49faac8bdacc9eb034a77 Mon Sep 17 00:00:00 2001 From: Adam Leventhal Date: Fri, 1 Dec 2023 15:51:43 -0800 Subject: [PATCH 03/16] move to openapiv3 v2.0.0-rc.1 (#4582) --- Cargo.lock | 39 ++++++++++++++------------------------- Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06c1c2b5b7..329f74bb77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1790,7 +1790,7 @@ dependencies = [ "omicron-test-utils", "omicron-workspace-hack", "openapi-lint", - "openapiv3 1.0.3", + "openapiv3", "pretty-hex 0.4.0", "schemars", "serde", @@ -1895,7 +1895,7 @@ dependencies = [ "hyper", "indexmap 2.1.0", "multer", - "openapiv3 2.0.0-rc.1", + "openapiv3", "paste", "percent-encoding", "proc-macro2", @@ -3245,7 +3245,7 @@ dependencies = [ "omicron-test-utils", "omicron-workspace-hack", "openapi-lint", - "openapiv3 1.0.3", + "openapiv3", "schemars", "serde", "serde_json", @@ -4039,7 +4039,7 @@ dependencies = [ "omicron-sled-agent", "omicron-test-utils", "omicron-workspace-hack", - "openapiv3 1.0.3", + "openapiv3", "openssl", "oso", "oximeter", @@ -4565,7 +4565,7 @@ dependencies = [ "omicron-workspace-hack", "once_cell", "openapi-lint", - "openapiv3 1.0.3", + "openapiv3", "schemars", "serde", "serde_json", @@ -4642,7 +4642,7 @@ dependencies = [ "omicron-workspace-hack", "once_cell", "openapi-lint", - "openapiv3 1.0.3", + "openapiv3", "openssl", "oxide-client", "oximeter", @@ -4840,7 +4840,7 @@ dependencies = [ "omicron-workspace-hack", "once_cell", "openapi-lint", - "openapiv3 1.0.3", + "openapiv3", "opte-ioctl", "oximeter", "oximeter-instruments", @@ -4970,7 +4970,7 @@ dependencies = [ "num-iter", "num-traits", "once_cell", - "openapiv3 2.0.0-rc.1", + "openapiv3", "petgraph", "postgres-types", "ppv-lite86", @@ -5066,26 +5066,15 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openapi-lint" version = "0.4.0" -source = "git+https://github.com/oxidecomputer/openapi-lint?branch=main#bb69a3a4a184d966bac2a0df2be5c9038d9867d0" +source = "git+https://github.com/oxidecomputer/openapi-lint?branch=main#ef442ee4343e97b6d9c217d3e7533962fe7d7236" dependencies = [ "heck 0.4.1", "indexmap 2.1.0", "lazy_static", - "openapiv3 1.0.3", + "openapiv3", "regex", ] -[[package]] -name = "openapiv3" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75e56d5c441965b6425165b7e3223cc933ca469834f4a8b4786817a1f9dc4f13" -dependencies = [ - "indexmap 2.1.0", - "serde", - "serde_json", -] - [[package]] name = "openapiv3" version = "2.0.0-rc.1" @@ -5312,7 +5301,7 @@ dependencies = [ "omicron-test-utils", "omicron-workspace-hack", "openapi-lint", - "openapiv3 1.0.3", + "openapiv3", "oximeter", "oximeter-client", "oximeter-db", @@ -6079,7 +6068,7 @@ dependencies = [ "heck 0.4.1", "http", "indexmap 2.1.0", - "openapiv3 2.0.0-rc.1", + "openapiv3", "proc-macro2", "quote", "regex", @@ -6097,7 +6086,7 @@ name = "progenitor-macro" version = "0.4.0" source = "git+https://github.com/oxidecomputer/progenitor?branch=main#9339b57628e1e76b1d7131ef93a6c0db2ab0a762" dependencies = [ - "openapiv3 2.0.0-rc.1", + "openapiv3", "proc-macro2", "progenitor-impl", "quote", @@ -9549,7 +9538,7 @@ dependencies = [ "omicron-test-utils", "omicron-workspace-hack", "openapi-lint", - "openapiv3 1.0.3", + "openapiv3", "rand 0.8.5", "reqwest", "schemars", diff --git a/Cargo.toml b/Cargo.toml index 6d47909399..533e710dc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -263,7 +263,7 @@ oxide-client = { path = "clients/oxide-client" } oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "258a8b59902dd36fc7ee5425e6b1fb5fc80d4649", features = [ "api", "std" ] } once_cell = "1.18.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } -openapiv3 = "1.0" +openapiv3 = "2.0.0-rc.1" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" From 6f2d1c5b1589295249e7b4548657374bfe11f909 Mon Sep 17 00:00:00 2001 From: Andy Fiddaman Date: Mon, 4 Dec 2023 16:55:29 +0000 Subject: [PATCH 04/16] Improve wtmp record mangling. (#4501) This fixes two problems with the current wtmp/utmp record mangling. First it is not idempotent, the system's notion of boot time is set to the current time of day whenever sled agent confirms that time is synchronised. Secondly, this is only approximate even for the first sled agent start, but it's plain wrong if sled agent restarts later. In conjunction with changes to stlouis, the `tmpx` utility is now able to process all zones itself, and uses the true system boot time for each zone when updating records. Fixes: https://github.com/oxidecomputer/omicron/issues/3514 --- sled-agent/src/services.rs | 45 ++++++-------------------- tools/install_builder_prerequisites.sh | 2 ++ 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 88f79e7064..dc309e8423 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -101,13 +101,11 @@ use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::BTreeMap; use std::collections::HashSet; -use std::iter; use std::iter::FromIterator; use std::net::{IpAddr, Ipv6Addr, SocketAddr}; use std::str::FromStr; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; use tokio::io::AsyncWriteExt; use tokio::sync::Mutex; use tokio::sync::{oneshot, MutexGuard}; @@ -2931,10 +2929,7 @@ impl ServiceManager { Ok(()) } - pub fn boottime_rewrite<'a>( - &self, - zones: impl Iterator, - ) { + pub fn boottime_rewrite(&self) { if self .inner .time_synced @@ -2945,33 +2940,13 @@ impl ServiceManager { return; } - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("SystemTime before UNIX EPOCH"); - - info!(self.inner.log, "Setting boot time to {:?}", now); - - let files: Vec = zones - .map(|z| z.root()) - .chain(iter::once(Utf8PathBuf::from("/"))) - .flat_map(|r| [r.join("var/adm/utmpx"), r.join("var/adm/wtmpx")]) - .collect(); - - for file in files { - let mut command = std::process::Command::new(PFEXEC); - let cmd = command.args(&[ - "/usr/platform/oxide/bin/tmpx", - &format!("{}", now.as_secs()), - &file.as_str(), - ]); - match execute(cmd) { - Err(e) => { - warn!(self.inner.log, "Updating {} failed: {}", &file, e); - } - Ok(_) => { - info!(self.inner.log, "Updated {}", &file); - } - } + // Call out to the 'tmpx' utility program which will rewrite the wtmpx + // and utmpx databases in every zone, including the global zone, to + // reflect the adjusted system boot time. + let mut command = std::process::Command::new(PFEXEC); + let cmd = command.args(&["/usr/platform/oxide/bin/tmpx", "-Z"]); + if let Err(e) = execute(cmd) { + warn!(self.inner.log, "Updating [wu]tmpx databases failed: {}", e); } } @@ -2980,7 +2955,7 @@ impl ServiceManager { if let Some(true) = self.inner.skip_timesync { info!(self.inner.log, "Configured to skip timesync checks"); - self.boottime_rewrite(existing_zones.values()); + self.boottime_rewrite(); return Ok(TimeSync { sync: true, ref_id: 0, @@ -3034,7 +3009,7 @@ impl ServiceManager { && correction.abs() <= 0.05; if sync { - self.boottime_rewrite(existing_zones.values()); + self.boottime_rewrite(); } Ok(TimeSync { diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index d3ecd8eaa8..1ce133dff3 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -131,6 +131,8 @@ function install_packages { "library/libxmlsec1" # "bindgen leverages libclang to preprocess, parse, and type check C and C++ header files." "pkg:/ooce/developer/clang-$CLANGVER" + "system/library/gcc-runtime" + "system/library/g++-runtime" ) # Install/update the set of packages. From 15115a461c1df915bd9b6f2e71855c126d8e4f6c Mon Sep 17 00:00:00 2001 From: Rain Date: Mon, 4 Dec 2023 12:33:07 -0800 Subject: [PATCH 05/16] [easy] [update-engine] change an info to a debug (#4603) Saw `wicket rack-update` print out this info message even though the group displayer would show a corresponding message anyway. --- update-engine/src/display/group_display.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update-engine/src/display/group_display.rs b/update-engine/src/display/group_display.rs index cfd37aac16..0e04361ce4 100644 --- a/update-engine/src/display/group_display.rs +++ b/update-engine/src/display/group_display.rs @@ -153,7 +153,7 @@ impl GroupDisplay { self.stats.apply_result(result); if result.before != result.after { - slog::info!( + slog::debug!( self.log, "add_event_report caused state transition"; "prefix" => &state.prefix, From c915eeb77fc153376a0734f712deaf1525a391f1 Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Mon, 4 Dec 2023 16:21:08 -0500 Subject: [PATCH 06/16] Detect and un-delete phantom disks (#4547) A "phantom" disk is a disk where the disk delete saga has run for it but unwound: this leaves that disk soft-deleted, but the resources and accounting for that disk remain. Users cannot request that the disk be deleted again, and it remains a phantom. There are two fixes for this: 1. Change the disk delete saga to undo the disk's soft delete and set the disk to faulted during an unwind. This way, users can request that disks be deleted repeatedly until it works. 2. Create a background task that detects these phantom disks and does the same thing: un-delete them and set them to faulted. This requires adding an index on `id` to the `disk` table, so the schema is bumped to 12.0.1. Fixes oxidecomputer/customer-support#58. --- common/src/nexus_config.rs | 24 +- dev-tools/omdb/src/bin/omdb/nexus.rs | 26 ++ dev-tools/omdb/tests/env.out | 12 + dev-tools/omdb/tests/successes.out | 12 + nexus/db-model/src/schema.rs | 6 +- nexus/db-queries/src/db/datastore/disk.rs | 290 +++++++++++++++++++++- nexus/examples/config.toml | 1 + nexus/src/app/background/init.rs | 23 +- nexus/src/app/background/mod.rs | 1 + nexus/src/app/background/phantom_disks.rs | 104 ++++++++ nexus/src/app/sagas/disk_delete.rs | 19 +- nexus/tests/config.test.toml | 1 + nexus/tests/integration_tests/disks.rs | 132 ++++++++++ schema/crdb/18.0.0/up01.sql | 4 + schema/crdb/dbinit.sql | 7 +- sled-agent/src/sim/storage.rs | 14 ++ smf/nexus/multi-sled/config-partial.toml | 1 + smf/nexus/single-sled/config-partial.toml | 1 + 18 files changed, 667 insertions(+), 11 deletions(-) create mode 100644 nexus/src/app/background/phantom_disks.rs create mode 100644 schema/crdb/18.0.0/up01.sql diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index 94c39b4436..740823e755 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -339,6 +339,8 @@ pub struct BackgroundTaskConfig { pub nat_cleanup: NatCleanupConfig, /// configuration for inventory tasks pub inventory: InventoryConfig, + /// configuration for phantom disks task + pub phantom_disks: PhantomDiskConfig, } #[serde_as] @@ -386,7 +388,7 @@ pub struct NatCleanupConfig { pub struct InventoryConfig { /// period (in seconds) for periodic activations of this background task /// - /// Each activation fetches information about all harware and software in + /// Each activation fetches information about all hardware and software in /// the system and inserts it into the database. This generates a moderate /// amount of data. #[serde_as(as = "DurationSeconds")] @@ -405,6 +407,14 @@ pub struct InventoryConfig { pub disable: bool, } +#[serde_as] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct PhantomDiskConfig { + /// period (in seconds) for periodic activations of this background task + #[serde_as(as = "DurationSeconds")] + pub period_secs: Duration, +} + /// Configuration for a nexus server #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct PackageConfig { @@ -508,8 +518,9 @@ mod test { BackgroundTaskConfig, Config, ConfigDropshotWithTls, ConsoleConfig, Database, DeploymentConfig, DnsTasksConfig, DpdConfig, ExternalEndpointsConfig, InternalDns, InventoryConfig, LoadError, - LoadErrorKind, MgdConfig, NatCleanupConfig, PackageConfig, SchemeName, - TimeseriesDbConfig, Tunables, UpdatesConfig, + LoadErrorKind, MgdConfig, NatCleanupConfig, PackageConfig, + PhantomDiskConfig, SchemeName, TimeseriesDbConfig, Tunables, + UpdatesConfig, }; use crate::address::{Ipv6Subnet, RACK_PREFIX}; use crate::api::internal::shared::SwitchLocation; @@ -663,6 +674,7 @@ mod test { inventory.period_secs = 10 inventory.nkeep = 11 inventory.disable = false + phantom_disks.period_secs = 30 [default_region_allocation_strategy] type = "random" seed = 0 @@ -764,7 +776,10 @@ mod test { period_secs: Duration::from_secs(10), nkeep: 11, disable: false, - } + }, + phantom_disks: PhantomDiskConfig { + period_secs: Duration::from_secs(30), + }, }, default_region_allocation_strategy: crate::nexus_config::RegionAllocationStrategy::Random { @@ -822,6 +837,7 @@ mod test { inventory.period_secs = 10 inventory.nkeep = 3 inventory.disable = false + phantom_disks.period_secs = 30 [default_region_allocation_strategy] type = "random" "##, diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index 9f91d38504..df5248b52d 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -515,6 +515,32 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) { ); } }; + } else if name == "phantom_disks" { + #[derive(Deserialize)] + struct TaskSuccess { + /// how many phantom disks were deleted ok + phantom_disk_deleted_ok: usize, + + /// how many phantom disks could not be deleted + phantom_disk_deleted_err: usize, + } + + match serde_json::from_value::(details.clone()) { + Err(error) => eprintln!( + "warning: failed to interpret task details: {:?}: {:?}", + error, details + ), + Ok(success) => { + println!( + " number of phantom disks deleted: {}", + success.phantom_disk_deleted_ok + ); + println!( + " number of phantom disk delete errors: {}", + success.phantom_disk_deleted_err + ); + } + }; } else { println!( "warning: unknown background task: {:?} \ diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index fd50d80c81..c08f592852 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -66,6 +66,10 @@ task: "nat_v4_garbage_collector" predetermined retention policy +task: "phantom_disks" + detects and un-deletes phantom disks + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT @@ -131,6 +135,10 @@ task: "nat_v4_garbage_collector" predetermined retention policy +task: "phantom_disks" + detects and un-deletes phantom disks + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. @@ -183,6 +191,10 @@ task: "nat_v4_garbage_collector" predetermined retention policy +task: "phantom_disks" + detects and un-deletes phantom disks + + --------------------------------------------- stderr: note: Nexus URL not specified. Will pick one from DNS. diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 6bc3a85e8a..65520ab59c 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -260,6 +260,10 @@ task: "nat_v4_garbage_collector" predetermined retention policy +task: "phantom_disks" + detects and un-deletes phantom disks + + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ @@ -357,6 +361,14 @@ task: "inventory_collection" last collection started: last collection done: +task: "phantom_disks" + configured period: every 30s + currently executing: no + last completed activation: iter 2, triggered by an explicit signal + started at (s ago) and ran for ms + number of phantom disks deleted: 0 + number of phantom disk delete errors: 0 + --------------------------------------------- stderr: note: using Nexus URL http://127.0.0.1:REDACTED_PORT/ diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index be345032ac..373785799e 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1301,7 +1301,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(17, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(18, 0, 0); allow_tables_to_appear_in_same_query!( system_update, @@ -1370,3 +1370,7 @@ allow_tables_to_appear_in_same_query!( switch_port, switch_port_settings_bgp_peer_config ); + +allow_tables_to_appear_in_same_query!(disk, virtual_provisioning_resource); + +allow_tables_to_appear_in_same_query!(volume, virtual_provisioning_resource); diff --git a/nexus/db-queries/src/db/datastore/disk.rs b/nexus/db-queries/src/db/datastore/disk.rs index a0d9bf12c3..26d439b350 100644 --- a/nexus/db-queries/src/db/datastore/disk.rs +++ b/nexus/db-queries/src/db/datastore/disk.rs @@ -25,11 +25,14 @@ use crate::db::model::DiskUpdate; use crate::db::model::Instance; use crate::db::model::Name; use crate::db::model::Project; +use crate::db::model::VirtualProvisioningResource; +use crate::db::model::Volume; use crate::db::pagination::paginated; use crate::db::queries::disk::DiskSetClauseForAttach; use crate::db::update_and_check::UpdateAndCheck; use crate::db::update_and_check::UpdateStatus; use async_bb8_diesel::AsyncRunQueryDsl; +use chrono::DateTime; use chrono::Utc; use diesel::prelude::*; use omicron_common::api; @@ -564,7 +567,7 @@ impl DataStore { /// Updates a disk record to indicate it has been deleted. /// - /// Returns the volume ID of associated with the deleted disk. + /// Returns the disk before any modifications are made by this function. /// /// Does not attempt to modify any resources (e.g. regions) which may /// belong to the disk. @@ -652,4 +655,289 @@ impl DataStore { } } } + + /// Set a disk to faulted and un-delete it + /// + /// If the disk delete saga unwinds, then the disk should _not_ remain + /// deleted: disk delete saga should be triggered again in order to fully + /// complete, and the only way to do that is to un-delete the disk. Set it + /// to faulted to ensure that it won't be used. + pub async fn project_undelete_disk_set_faulted_no_auth( + &self, + disk_id: &Uuid, + ) -> Result<(), Error> { + use db::schema::disk::dsl; + let conn = self.pool_connection_unauthorized().await?; + + let faulted = api::external::DiskState::Faulted.label(); + + let result = diesel::update(dsl::disk) + .filter(dsl::time_deleted.is_not_null()) + .filter(dsl::id.eq(*disk_id)) + .set(( + dsl::time_deleted.eq(None::>), + dsl::disk_state.eq(faulted), + )) + .check_if_exists::(*disk_id) + .execute_and_check(&conn) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Disk, + LookupType::ById(*disk_id), + ), + ) + })?; + + match result.status { + UpdateStatus::Updated => Ok(()), + UpdateStatus::NotUpdatedButExists => { + let disk = result.found; + let disk_state = disk.state(); + + if disk.time_deleted().is_none() + && disk_state.state() == &api::external::DiskState::Faulted + { + // To maintain idempotency, if the disk has already been + // faulted, don't throw an error. + return Ok(()); + } else { + // NOTE: This is a "catch-all" error case, more specific + // errors should be preferred as they're more actionable. + return Err(Error::InternalError { + internal_message: String::from( + "disk exists, but cannot be faulted", + ), + }); + } + } + } + } + + /// Find disks that have been deleted but still have a + /// `virtual_provisioning_resource` record: this indicates that a disk + /// delete saga partially succeeded, then unwound, which (before the fixes + /// in customer-support#58) would mean the disk was deleted but the project + /// it was in could not be deleted (due to an erroneous number of bytes + /// "still provisioned"). + pub async fn find_phantom_disks(&self) -> ListResultVec { + use db::schema::disk::dsl; + use db::schema::virtual_provisioning_resource::dsl as resource_dsl; + use db::schema::volume::dsl as volume_dsl; + + let conn = self.pool_connection_unauthorized().await?; + + let potential_phantom_disks: Vec<( + Disk, + Option, + Option, + )> = dsl::disk + .filter(dsl::time_deleted.is_not_null()) + .left_join( + resource_dsl::virtual_provisioning_resource + .on(resource_dsl::id.eq(dsl::id)), + ) + .left_join(volume_dsl::volume.on(dsl::volume_id.eq(volume_dsl::id))) + .select(( + Disk::as_select(), + Option::::as_select(), + Option::::as_select(), + )) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + // The first forward steps of the disk delete saga (plus the volume + // delete sub saga) are as follows: + // + // 1. soft-delete the disk + // 2. call virtual_provisioning_collection_delete_disk + // 3. soft-delete the disk's volume + // + // Before the fixes as part of customer-support#58, steps 1 and 3 did + // not have undo steps, where step 2 did. In order to detect when the + // disk delete saga unwound, find entries where + // + // 1. the disk and volume are soft-deleted + // 2. the `virtual_provisioning_resource` exists + // + // It's important not to conflict with any currently running disk delete + // saga. + + Ok(potential_phantom_disks + .into_iter() + .filter(|(disk, resource, volume)| { + if let Some(volume) = volume { + // In this branch, the volume record exists. Because it was + // returned by the query above, if it is soft-deleted we + // then know the saga unwound before the volume record could + // be hard deleted. This won't conflict with a running disk + // delete saga, because the resource record should be None + // if the disk and volume were already soft deleted (if + // there is one, the saga will be at or past step 3). + disk.time_deleted().is_some() + && volume.time_deleted.is_some() + && resource.is_some() + } else { + // In this branch, the volume record was hard-deleted. The + // saga could still have unwound after hard deleting the + // volume record, so proceed with filtering. This won't + // conflict with a running disk delete saga because the + // resource record should be None if the disk was soft + // deleted and the volume was hard deleted (if there is one, + // the saga should be almost finished as the volume hard + // delete is the last thing it does). + disk.time_deleted().is_some() && resource.is_some() + } + }) + .map(|(disk, _, _)| disk) + .collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::db::datastore::datastore_test; + use nexus_test_utils::db::test_setup_database; + use nexus_types::external_api::params; + use omicron_common::api::external; + use omicron_test_utils::dev; + + #[tokio::test] + async fn test_undelete_disk_set_faulted_idempotent() { + let logctx = + dev::test_setup_log("test_undelete_disk_set_faulted_idempotent"); + let log = logctx.log.new(o!()); + let mut db = test_setup_database(&log).await; + let (opctx, db_datastore) = datastore_test(&logctx, &db).await; + + let silo_id = opctx.authn.actor().unwrap().silo_id().unwrap(); + + let (authz_project, _db_project) = db_datastore + .project_create( + &opctx, + Project::new( + silo_id, + params::ProjectCreate { + identity: external::IdentityMetadataCreateParams { + name: "testpost".parse().unwrap(), + description: "please ignore".to_string(), + }, + }, + ), + ) + .await + .unwrap(); + + let disk = db_datastore + .project_create_disk( + &opctx, + &authz_project, + Disk::new( + Uuid::new_v4(), + authz_project.id(), + Uuid::new_v4(), + params::DiskCreate { + identity: external::IdentityMetadataCreateParams { + name: "first-post".parse().unwrap(), + description: "just trying things out".to_string(), + }, + disk_source: params::DiskSource::Blank { + block_size: params::BlockSize::try_from(512) + .unwrap(), + }, + size: external::ByteCount::from(2147483648), + }, + db::model::BlockSize::Traditional, + DiskRuntimeState::new(), + ) + .unwrap(), + ) + .await + .unwrap(); + + let (.., authz_disk, db_disk) = LookupPath::new(&opctx, &db_datastore) + .disk_id(disk.id()) + .fetch() + .await + .unwrap(); + + db_datastore + .disk_update_runtime( + &opctx, + &authz_disk, + &db_disk.runtime().detach(), + ) + .await + .unwrap(); + + db_datastore + .project_delete_disk_no_auth( + &authz_disk.id(), + &[external::DiskState::Detached], + ) + .await + .unwrap(); + + // Assert initial state - deleting the Disk will make LookupPath::fetch + // not work. + { + LookupPath::new(&opctx, &db_datastore) + .disk_id(disk.id()) + .fetch() + .await + .unwrap_err(); + } + + // Function under test: call this twice to ensure it's idempotent + + db_datastore + .project_undelete_disk_set_faulted_no_auth(&authz_disk.id()) + .await + .unwrap(); + + // Assert state change + + { + let (.., db_disk) = LookupPath::new(&opctx, &db_datastore) + .disk_id(disk.id()) + .fetch() + .await + .unwrap(); + + assert!(db_disk.time_deleted().is_none()); + assert_eq!( + db_disk.runtime().disk_state, + external::DiskState::Faulted.label().to_string() + ); + } + + db_datastore + .project_undelete_disk_set_faulted_no_auth(&authz_disk.id()) + .await + .unwrap(); + + // Assert state is the same after the second call + + { + let (.., db_disk) = LookupPath::new(&opctx, &db_datastore) + .disk_id(disk.id()) + .fetch() + .await + .unwrap(); + + assert!(db_disk.time_deleted().is_none()); + assert_eq!( + db_disk.runtime().disk_state, + external::DiskState::Faulted.label().to_string() + ); + } + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } } diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index 3679fa8196..9d6bf2d22f 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -100,6 +100,7 @@ inventory.period_secs = 600 inventory.nkeep = 5 # Disable inventory collection altogether (for emergencies) inventory.disable = false +phantom_disks.period_secs = 30 [default_region_allocation_strategy] # allocate region on 3 random distinct zpools, on 3 random distinct sleds. diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index d27248ffdc..cfa023a013 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -11,6 +11,7 @@ use super::dns_servers; use super::external_endpoints; use super::inventory_collection; use super::nat_cleanup; +use super::phantom_disks; use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; @@ -52,6 +53,9 @@ pub struct BackgroundTasks { /// task handle for the task that collects inventory pub task_inventory_collection: common::TaskHandle, + + /// task handle for the task that detects phantom disks + pub task_phantom_disks: common::TaskHandle, } impl BackgroundTasks { @@ -122,7 +126,7 @@ impl BackgroundTasks { // Background task: inventory collector let task_inventory_collection = { let collector = inventory_collection::InventoryCollector::new( - datastore, + datastore.clone(), resolver, &nexus_id.to_string(), config.inventory.nkeep, @@ -143,6 +147,22 @@ impl BackgroundTasks { task }; + // Background task: phantom disk detection + let task_phantom_disks = { + let detector = phantom_disks::PhantomDiskDetector::new(datastore); + + let task = driver.register( + String::from("phantom_disks"), + String::from("detects and un-deletes phantom disks"), + config.phantom_disks.period_secs, + Box::new(detector), + opctx.child(BTreeMap::new()), + vec![], + ); + + task + }; + BackgroundTasks { driver, task_internal_dns_config, @@ -153,6 +173,7 @@ impl BackgroundTasks { external_endpoints, nat_cleanup, task_inventory_collection, + task_phantom_disks, } } diff --git a/nexus/src/app/background/mod.rs b/nexus/src/app/background/mod.rs index 954207cb3c..70b20224d4 100644 --- a/nexus/src/app/background/mod.rs +++ b/nexus/src/app/background/mod.rs @@ -12,6 +12,7 @@ mod external_endpoints; mod init; mod inventory_collection; mod nat_cleanup; +mod phantom_disks; mod status; pub use common::Driver; diff --git a/nexus/src/app/background/phantom_disks.rs b/nexus/src/app/background/phantom_disks.rs new file mode 100644 index 0000000000..b038d70ac6 --- /dev/null +++ b/nexus/src/app/background/phantom_disks.rs @@ -0,0 +1,104 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Background task for detecting and un-deleting phantom disks +//! +//! A "phantom" disk is one where a disk delete saga partially completed but +//! unwound: before a fix for customer-support#58, this would leave disks +//! deleted but would also leave a `virtual_provisioning_resource` record for +//! that disk. There would be no way to re-trigger the disk delete saga as the +//! disk was deleted, so the project that disk was in could not be deleted +//! because associated virtual provisioning resources were still being consumed. +//! +//! The fix for customer-support#58 changes the disk delete saga's unwind to +//! also un-delete the disk and set it to faulted. This enables it to be deleted +//! again. Correcting the disk delete saga's unwind means that phantom disks +//! will not be created in the future when the disk delete saga unwinds, but +//! this background task is required to apply the same fix for disks that are +//! already in this phantom state. + +use super::common::BackgroundTask; +use futures::future::BoxFuture; +use futures::FutureExt; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use serde_json::json; +use std::sync::Arc; + +pub struct PhantomDiskDetector { + datastore: Arc, +} + +impl PhantomDiskDetector { + pub fn new(datastore: Arc) -> Self { + PhantomDiskDetector { datastore } + } +} + +impl BackgroundTask for PhantomDiskDetector { + fn activate<'a, 'b, 'c>( + &'a mut self, + opctx: &'b OpContext, + ) -> BoxFuture<'c, serde_json::Value> + where + 'a: 'c, + 'b: 'c, + { + async { + let log = &opctx.log; + warn!(&log, "phantom disk task started"); + + let phantom_disks = match self.datastore.find_phantom_disks().await + { + Ok(phantom_disks) => phantom_disks, + Err(e) => { + warn!(&log, "error from find_phantom_disks: {:?}", e); + return json!({ + "error": + format!("failed find_phantom_disks: {:#}", e) + }); + } + }; + + let mut phantom_disk_deleted_ok = 0; + let mut phantom_disk_deleted_err = 0; + + for disk in phantom_disks { + warn!(&log, "phantom disk {} found!", disk.id()); + + // If a phantom disk is found, then un-delete it and set it to + // faulted: this will allow a user to request deleting it again. + + let result = self + .datastore + .project_undelete_disk_set_faulted_no_auth(&disk.id()) + .await; + + if let Err(e) = result { + error!( + &log, + "error un-deleting disk {} and setting to faulted: {:#}", + disk.id(), + e, + ); + phantom_disk_deleted_err += 1; + } else { + info!( + &log, + "phandom disk {} un-deleted andset to faulted ok", + disk.id(), + ); + phantom_disk_deleted_ok += 1; + } + } + + warn!(&log, "phantom disk task done"); + json!({ + "phantom_disk_deleted_ok": phantom_disk_deleted_ok, + "phantom_disk_deleted_err": phantom_disk_deleted_err, + }) + } + .boxed() + } +} diff --git a/nexus/src/app/sagas/disk_delete.rs b/nexus/src/app/sagas/disk_delete.rs index f2d80d64f5..8f6d74da0a 100644 --- a/nexus/src/app/sagas/disk_delete.rs +++ b/nexus/src/app/sagas/disk_delete.rs @@ -32,10 +32,8 @@ pub(crate) struct Params { declare_saga_actions! { disk_delete; DELETE_DISK_RECORD -> "deleted_disk" { - // TODO: See the comment on the "DeleteRegions" step, - // we may want to un-delete the disk if we cannot remove - // underlying regions. + sdd_delete_disk_record + - sdd_delete_disk_record_undo } SPACE_ACCOUNT -> "no_result1" { + sdd_account_space @@ -117,6 +115,21 @@ async fn sdd_delete_disk_record( Ok(disk) } +async fn sdd_delete_disk_record_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + + osagactx + .datastore() + .project_undelete_disk_set_faulted_no_auth(¶ms.disk_id) + .await + .map_err(ActionError::action_failed)?; + + Ok(()) +} + async fn sdd_account_space( sagactx: NexusActionContext, ) -> Result<(), ActionError> { diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index fbed9aed8e..a4436234f0 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -98,6 +98,7 @@ inventory.period_secs = 600 inventory.nkeep = 3 # Disable inventory collection altogether (for emergencies) inventory.disable = false +phantom_disks.period_secs = 30 [default_region_allocation_strategy] # we only have one sled in the test environment, so we need to use the diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index a5a8339c34..f7403275b1 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -1241,6 +1241,138 @@ async fn test_disk_virtual_provisioning_collection( ); } +#[nexus_test] +async fn test_disk_virtual_provisioning_collection_failed_delete( + cptestctx: &ControlPlaneTestContext, +) { + // Confirm that there's a panic deleting a project if a disk deletion fails + let client = &cptestctx.external_client; + let nexus = &cptestctx.server.apictx().nexus; + let datastore = nexus.datastore(); + + let disk_test = DiskTest::new(&cptestctx).await; + + populate_ip_pool(&client, "default", None).await; + let project_id1 = create_project(client, PROJECT_NAME).await.identity.id; + + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + // Create a 1 GB disk + let disk_size = ByteCount::from_gibibytes_u32(1); + let disks_url = get_disks_url(); + let disk_one = params::DiskCreate { + identity: IdentityMetadataCreateParams { + name: "disk-one".parse().unwrap(), + description: String::from("sells rainsticks"), + }, + disk_source: params::DiskSource::Blank { + block_size: params::BlockSize::try_from(512).unwrap(), + }, + size: disk_size, + }; + NexusRequest::new( + RequestBuilder::new(client, Method::POST, &disks_url) + .body(Some(&disk_one)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("unexpected failure creating 1 GiB disk"); + + // Assert correct virtual provisioning collection numbers + let virtual_provisioning_collection = datastore + .virtual_provisioning_collection_get(&opctx, project_id1) + .await + .unwrap(); + assert_eq!( + virtual_provisioning_collection.virtual_disk_bytes_provisioned.0, + disk_size + ); + + // Set the third agent to fail when deleting regions + let zpool = &disk_test.zpools[2]; + let dataset = &zpool.datasets[0]; + disk_test + .sled_agent + .get_crucible_dataset(zpool.id, dataset.id) + .await + .set_region_deletion_error(true) + .await; + + // Delete the disk - expect this to fail + let disk_url = format!("/v1/disks/{}?project={}", "disk-one", PROJECT_NAME); + + NexusRequest::new( + RequestBuilder::new(client, Method::DELETE, &disk_url) + .expect_status(Some(StatusCode::INTERNAL_SERVER_ERROR)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("unexpected success deleting 1 GiB disk"); + + // The virtual provisioning collection numbers haven't changed + let virtual_provisioning_collection = datastore + .virtual_provisioning_collection_get(&opctx, project_id1) + .await + .unwrap(); + assert_eq!( + virtual_provisioning_collection.virtual_disk_bytes_provisioned.0, + disk_size + ); + + // And the disk is now faulted + let disk = disk_get(&client, &disk_url).await; + assert_eq!(disk.state, DiskState::Faulted); + + // Set the third agent to respond normally + disk_test + .sled_agent + .get_crucible_dataset(zpool.id, dataset.id) + .await + .set_region_deletion_error(false) + .await; + + // Request disk delete again + NexusRequest::new( + RequestBuilder::new(client, Method::DELETE, &disk_url) + .expect_status(Some(StatusCode::NO_CONTENT)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("unexpected failure deleting 1 GiB disk"); + + // Delete the project's default VPC subnet and VPC + let subnet_url = + format!("/v1/vpc-subnets/default?project={}&vpc=default", PROJECT_NAME); + NexusRequest::object_delete(&client, &subnet_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to make request"); + + let vpc_url = format!("/v1/vpcs/default?project={}", PROJECT_NAME); + NexusRequest::object_delete(&client, &vpc_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to make request"); + + // The project can be deleted now + let url = format!("/v1/projects/{}", PROJECT_NAME); + NexusRequest::new( + RequestBuilder::new(client, Method::DELETE, &url) + .expect_status(Some(StatusCode::NO_CONTENT)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("unexpected failure deleting project"); +} + // Test disk size accounting #[nexus_test] async fn test_disk_size_accounting(cptestctx: &ControlPlaneTestContext) { diff --git a/schema/crdb/18.0.0/up01.sql b/schema/crdb/18.0.0/up01.sql new file mode 100644 index 0000000000..018bb36dcb --- /dev/null +++ b/schema/crdb/18.0.0/up01.sql @@ -0,0 +1,4 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_deleted_disk ON omicron.public.disk ( + id +) WHERE + time_deleted IS NOT NULL; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index f4caa2a4e6..f82829a2d9 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -1026,6 +1026,11 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_disk_by_instance ON omicron.public.disk ) WHERE time_deleted IS NULL AND attach_instance_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS lookup_deleted_disk ON omicron.public.disk ( + id +) WHERE + time_deleted IS NOT NULL; + CREATE TABLE IF NOT EXISTS omicron.public.image ( /* Identity metadata (resource) */ id UUID PRIMARY KEY, @@ -3009,7 +3014,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '17.0.0', NULL) + ( TRUE, NOW(), NOW(), '18.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 2528a258d7..101228934d 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -40,6 +40,7 @@ struct CrucibleDataInner { running_snapshots: HashMap>, on_create: Option, region_creation_error: bool, + region_deletion_error: bool, creating_a_running_snapshot_should_fail: bool, next_port: u16, } @@ -53,6 +54,7 @@ impl CrucibleDataInner { running_snapshots: HashMap::new(), on_create: None, region_creation_error: false, + region_deletion_error: false, creating_a_running_snapshot_should_fail: false, next_port: crucible_port, } @@ -129,6 +131,10 @@ impl CrucibleDataInner { ); } + if self.region_deletion_error { + bail!("region deletion error!"); + } + let id = Uuid::from_str(&id.0).unwrap(); if let Some(region) = self.regions.get_mut(&id) { if region.state == State::Failed { @@ -229,6 +235,10 @@ impl CrucibleDataInner { self.region_creation_error = value; } + fn set_region_deletion_error(&mut self, value: bool) { + self.region_deletion_error = value; + } + fn create_running_snapshot( &mut self, id: &RegionId, @@ -391,6 +401,10 @@ impl CrucibleData { self.inner.lock().await.set_region_creation_error(value); } + pub async fn set_region_deletion_error(&self, value: bool) { + self.inner.lock().await.set_region_deletion_error(value); + } + pub async fn create_running_snapshot( &self, id: &RegionId, diff --git a/smf/nexus/multi-sled/config-partial.toml b/smf/nexus/multi-sled/config-partial.toml index 94c8f5572e..d330f32ab6 100644 --- a/smf/nexus/multi-sled/config-partial.toml +++ b/smf/nexus/multi-sled/config-partial.toml @@ -46,6 +46,7 @@ inventory.period_secs = 600 inventory.nkeep = 3 # Disable inventory collection altogether (for emergencies) inventory.disable = false +phantom_disks.period_secs = 30 [default_region_allocation_strategy] # by default, allocate across 3 distinct sleds diff --git a/smf/nexus/single-sled/config-partial.toml b/smf/nexus/single-sled/config-partial.toml index fcaa6176a8..cbd4851613 100644 --- a/smf/nexus/single-sled/config-partial.toml +++ b/smf/nexus/single-sled/config-partial.toml @@ -46,6 +46,7 @@ inventory.period_secs = 600 inventory.nkeep = 3 # Disable inventory collection altogether (for emergencies) inventory.disable = false +phantom_disks.period_secs = 30 [default_region_allocation_strategy] # by default, allocate without requirement for distinct sleds. From 9b666e73dec06f2a645a714cdfbb21d63a2f3504 Mon Sep 17 00:00:00 2001 From: bnaecker Date: Mon, 4 Dec 2023 13:42:45 -0800 Subject: [PATCH 07/16] Support missing samples in oximeter (#4552) - Add a `Datum::Missing` and `MissingDatum`, which records the intended datum type and an optional start time for a sample which could not be produced. - Database upgrades which make all scalar datum columns Nullable. Array fields are _not_ made Nullable, since ClickHouse doesn't support composite types like arrays inside a Nullable wrapper type. The empty array is used as a sentinel, which is OK since we can't have zero-length array histograms in Oximeter. Add a test which will fail if we ever change that. - Rework database serialization to handle Nullable types or empty arrays. This uses a new helper trait to convert a NULL (which has no type information) to the intended datum type, or an empty array to a histogram. - Add a test for each measurement type that we can recover a missing sample of that type -- NULLs for scalar values and empty arrays for histograms. --- Cargo.lock | 1 + openapi/nexus-internal.json | 119 ++++ openapi/nexus.json | 67 +++ openapi/sled-agent.json | 119 ++++ oximeter/db/notes.txt | 232 -------- oximeter/db/schema/replicated/4/up01.sql | 1 + oximeter/db/schema/replicated/4/up02.sql | 1 + oximeter/db/schema/replicated/4/up03.sql | 1 + oximeter/db/schema/replicated/4/up04.sql | 1 + oximeter/db/schema/replicated/4/up05.sql | 1 + oximeter/db/schema/replicated/4/up06.sql | 1 + oximeter/db/schema/replicated/4/up07.sql | 1 + oximeter/db/schema/replicated/4/up08.sql | 1 + oximeter/db/schema/replicated/4/up09.sql | 1 + oximeter/db/schema/replicated/4/up10.sql | 1 + oximeter/db/schema/replicated/4/up11.sql | 1 + oximeter/db/schema/replicated/4/up12.sql | 1 + oximeter/db/schema/replicated/4/up13.sql | 1 + oximeter/db/schema/replicated/4/up14.sql | 1 + oximeter/db/schema/replicated/4/up15.sql | 1 + oximeter/db/schema/replicated/4/up16.sql | 1 + oximeter/db/schema/replicated/4/up17.sql | 1 + oximeter/db/schema/replicated/4/up18.sql | 1 + oximeter/db/schema/replicated/4/up19.sql | 1 + oximeter/db/schema/replicated/4/up20.sql | 1 + oximeter/db/schema/replicated/4/up21.sql | 1 + oximeter/db/schema/replicated/4/up22.sql | 1 + oximeter/db/schema/replicated/4/up23.sql | 1 + oximeter/db/schema/replicated/4/up24.sql | 1 + oximeter/db/schema/replicated/4/up25.sql | 1 + oximeter/db/schema/replicated/4/up26.sql | 1 + oximeter/db/schema/replicated/4/up27.sql | 1 + oximeter/db/schema/replicated/4/up28.sql | 1 + oximeter/db/schema/replicated/4/up29.sql | 1 + oximeter/db/schema/replicated/4/up30.sql | 1 + oximeter/db/schema/replicated/4/up31.sql | 1 + oximeter/db/schema/replicated/4/up32.sql | 1 + oximeter/db/schema/replicated/db-init.sql | 64 +-- oximeter/db/schema/single-node/4/up01.sql | 9 + oximeter/db/schema/single-node/4/up02.sql | 1 + oximeter/db/schema/single-node/4/up03.sql | 1 + oximeter/db/schema/single-node/4/up04.sql | 1 + oximeter/db/schema/single-node/4/up05.sql | 1 + oximeter/db/schema/single-node/4/up06.sql | 1 + oximeter/db/schema/single-node/4/up07.sql | 1 + oximeter/db/schema/single-node/4/up08.sql | 1 + oximeter/db/schema/single-node/4/up09.sql | 1 + oximeter/db/schema/single-node/4/up10.sql | 1 + oximeter/db/schema/single-node/4/up11.sql | 1 + oximeter/db/schema/single-node/4/up12.sql | 1 + oximeter/db/schema/single-node/4/up13.sql | 1 + oximeter/db/schema/single-node/4/up14.sql | 1 + oximeter/db/schema/single-node/4/up15.sql | 1 + oximeter/db/schema/single-node/4/up16.sql | 1 + oximeter/db/schema/single-node/db-init.sql | 49 +- oximeter/db/src/client.rs | 195 ++++--- oximeter/db/src/model.rs | 636 ++++++++++++++++++--- oximeter/oximeter/Cargo.toml | 1 + oximeter/oximeter/src/histogram.rs | 13 +- oximeter/oximeter/src/test_util.rs | 12 +- oximeter/oximeter/src/traits.rs | 36 +- oximeter/oximeter/src/types.rs | 105 +++- 62 files changed, 1255 insertions(+), 450 deletions(-) delete mode 100644 oximeter/db/notes.txt create mode 100644 oximeter/db/schema/replicated/4/up01.sql create mode 100644 oximeter/db/schema/replicated/4/up02.sql create mode 100644 oximeter/db/schema/replicated/4/up03.sql create mode 100644 oximeter/db/schema/replicated/4/up04.sql create mode 100644 oximeter/db/schema/replicated/4/up05.sql create mode 100644 oximeter/db/schema/replicated/4/up06.sql create mode 100644 oximeter/db/schema/replicated/4/up07.sql create mode 100644 oximeter/db/schema/replicated/4/up08.sql create mode 100644 oximeter/db/schema/replicated/4/up09.sql create mode 100644 oximeter/db/schema/replicated/4/up10.sql create mode 100644 oximeter/db/schema/replicated/4/up11.sql create mode 100644 oximeter/db/schema/replicated/4/up12.sql create mode 100644 oximeter/db/schema/replicated/4/up13.sql create mode 100644 oximeter/db/schema/replicated/4/up14.sql create mode 100644 oximeter/db/schema/replicated/4/up15.sql create mode 100644 oximeter/db/schema/replicated/4/up16.sql create mode 100644 oximeter/db/schema/replicated/4/up17.sql create mode 100644 oximeter/db/schema/replicated/4/up18.sql create mode 100644 oximeter/db/schema/replicated/4/up19.sql create mode 100644 oximeter/db/schema/replicated/4/up20.sql create mode 100644 oximeter/db/schema/replicated/4/up21.sql create mode 100644 oximeter/db/schema/replicated/4/up22.sql create mode 100644 oximeter/db/schema/replicated/4/up23.sql create mode 100644 oximeter/db/schema/replicated/4/up24.sql create mode 100644 oximeter/db/schema/replicated/4/up25.sql create mode 100644 oximeter/db/schema/replicated/4/up26.sql create mode 100644 oximeter/db/schema/replicated/4/up27.sql create mode 100644 oximeter/db/schema/replicated/4/up28.sql create mode 100644 oximeter/db/schema/replicated/4/up29.sql create mode 100644 oximeter/db/schema/replicated/4/up30.sql create mode 100644 oximeter/db/schema/replicated/4/up31.sql create mode 100644 oximeter/db/schema/replicated/4/up32.sql create mode 100644 oximeter/db/schema/single-node/4/up01.sql create mode 100644 oximeter/db/schema/single-node/4/up02.sql create mode 100644 oximeter/db/schema/single-node/4/up03.sql create mode 100644 oximeter/db/schema/single-node/4/up04.sql create mode 100644 oximeter/db/schema/single-node/4/up05.sql create mode 100644 oximeter/db/schema/single-node/4/up06.sql create mode 100644 oximeter/db/schema/single-node/4/up07.sql create mode 100644 oximeter/db/schema/single-node/4/up08.sql create mode 100644 oximeter/db/schema/single-node/4/up09.sql create mode 100644 oximeter/db/schema/single-node/4/up10.sql create mode 100644 oximeter/db/schema/single-node/4/up11.sql create mode 100644 oximeter/db/schema/single-node/4/up12.sql create mode 100644 oximeter/db/schema/single-node/4/up13.sql create mode 100644 oximeter/db/schema/single-node/4/up14.sql create mode 100644 oximeter/db/schema/single-node/4/up15.sql create mode 100644 oximeter/db/schema/single-node/4/up16.sql diff --git a/Cargo.lock b/Cargo.lock index 329f74bb77..28d3015025 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5261,6 +5261,7 @@ dependencies = [ "rstest", "schemars", "serde", + "serde_json", "strum", "thiserror", "trybuild", diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 7785d232d9..caf1414f53 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -2568,9 +2568,60 @@ "datum", "type" ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/MissingDatum" + }, + "type": { + "type": "string", + "enum": [ + "missing" + ] + } + }, + "required": [ + "datum", + "type" + ] } ] }, + "DatumType": { + "description": "The type of an individual datum of a metric.", + "type": "string", + "enum": [ + "bool", + "i8", + "u8", + "i16", + "u16", + "i32", + "u32", + "i64", + "u64", + "f32", + "f64", + "string", + "bytes", + "cumulative_i64", + "cumulative_u64", + "cumulative_f32", + "cumulative_f64", + "histogram_i8", + "histogram_u8", + "histogram_i16", + "histogram_u16", + "histogram_i32", + "histogram_u32", + "histogram_i64", + "histogram_u64", + "histogram_f32", + "histogram_f64" + ] + }, "DiskRuntimeState": { "description": "Runtime state of the Disk, which includes its attach state and some minimal metadata", "type": "object", @@ -4128,9 +4179,77 @@ "content", "type" ] + }, + { + "type": "object", + "properties": { + "content": { + "type": "object", + "properties": { + "datum_type": { + "$ref": "#/components/schemas/DatumType" + } + }, + "required": [ + "datum_type" + ] + }, + "type": { + "type": "string", + "enum": [ + "missing_datum_requires_start_time" + ] + } + }, + "required": [ + "content", + "type" + ] + }, + { + "type": "object", + "properties": { + "content": { + "type": "object", + "properties": { + "datum_type": { + "$ref": "#/components/schemas/DatumType" + } + }, + "required": [ + "datum_type" + ] + }, + "type": { + "type": "string", + "enum": [ + "missing_datum_cannot_have_start_time" + ] + } + }, + "required": [ + "content", + "type" + ] } ] }, + "MissingDatum": { + "type": "object", + "properties": { + "datum_type": { + "$ref": "#/components/schemas/DatumType" + }, + "start_time": { + "nullable": true, + "type": "string", + "format": "date-time" + } + }, + "required": [ + "datum_type" + ] + }, "Name": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", diff --git a/openapi/nexus.json b/openapi/nexus.json index 15e75f93ff..a6dffc6265 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -9742,9 +9742,60 @@ "datum", "type" ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/MissingDatum" + }, + "type": { + "type": "string", + "enum": [ + "missing" + ] + } + }, + "required": [ + "datum", + "type" + ] } ] }, + "DatumType": { + "description": "The type of an individual datum of a metric.", + "type": "string", + "enum": [ + "bool", + "i8", + "u8", + "i16", + "u16", + "i32", + "u32", + "i64", + "u64", + "f32", + "f64", + "string", + "bytes", + "cumulative_i64", + "cumulative_u64", + "cumulative_f32", + "cumulative_f64", + "histogram_i8", + "histogram_u8", + "histogram_i16", + "histogram_u16", + "histogram_i32", + "histogram_u32", + "histogram_i64", + "histogram_u64", + "histogram_f32", + "histogram_f64" + ] + }, "DerEncodedKeyPair": { "type": "object", "properties": { @@ -12269,6 +12320,22 @@ "items" ] }, + "MissingDatum": { + "type": "object", + "properties": { + "datum_type": { + "$ref": "#/components/schemas/DatumType" + }, + "start_time": { + "nullable": true, + "type": "string", + "format": "date-time" + } + }, + "required": [ + "datum_type" + ] + }, "Name": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 9951392e98..5e217b27a4 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -2898,9 +2898,60 @@ "datum", "type" ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/MissingDatum" + }, + "type": { + "type": "string", + "enum": [ + "missing" + ] + } + }, + "required": [ + "datum", + "type" + ] } ] }, + "DatumType": { + "description": "The type of an individual datum of a metric.", + "type": "string", + "enum": [ + "bool", + "i8", + "u8", + "i16", + "u16", + "i32", + "u32", + "i64", + "u64", + "f32", + "f64", + "string", + "bytes", + "cumulative_i64", + "cumulative_u64", + "cumulative_f32", + "cumulative_f64", + "histogram_i8", + "histogram_u8", + "histogram_i16", + "histogram_u16", + "histogram_i32", + "histogram_u32", + "histogram_i64", + "histogram_u64", + "histogram_f32", + "histogram_f64" + ] + }, "DeleteVirtualNetworkInterfaceHost": { "description": "The data needed to identify a virtual IP for which a sled maintains an OPTE virtual-to-physical mapping such that that mapping can be deleted.", "type": "object", @@ -4819,9 +4870,77 @@ "content", "type" ] + }, + { + "type": "object", + "properties": { + "content": { + "type": "object", + "properties": { + "datum_type": { + "$ref": "#/components/schemas/DatumType" + } + }, + "required": [ + "datum_type" + ] + }, + "type": { + "type": "string", + "enum": [ + "missing_datum_requires_start_time" + ] + } + }, + "required": [ + "content", + "type" + ] + }, + { + "type": "object", + "properties": { + "content": { + "type": "object", + "properties": { + "datum_type": { + "$ref": "#/components/schemas/DatumType" + } + }, + "required": [ + "datum_type" + ] + }, + "type": { + "type": "string", + "enum": [ + "missing_datum_cannot_have_start_time" + ] + } + }, + "required": [ + "content", + "type" + ] } ] }, + "MissingDatum": { + "type": "object", + "properties": { + "datum_type": { + "$ref": "#/components/schemas/DatumType" + }, + "start_time": { + "nullable": true, + "type": "string", + "format": "date-time" + } + }, + "required": [ + "datum_type" + ] + }, "Name": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", diff --git a/oximeter/db/notes.txt b/oximeter/db/notes.txt deleted file mode 100644 index 66c3871d46..0000000000 --- a/oximeter/db/notes.txt +++ /dev/null @@ -1,232 +0,0 @@ -Some notes on querying - -For pagination: - -- Timeseries name is enough for paginated list timeseries endpoint. -It's just normal keyset pagination. - -- For the timeseries data, we'll be using limit/offset pagination. We'll -run the query to get the consistent timeseries keys each time. This is -the `ScanParams` part of the `WhichPage`. The `PageSelector` is the offset. - - -Now, how to run more complex queries? A good example is something like, -aggregating the timeseries across all but one field. For example, let's -look at the Nexus HTTP latency data. The fields are: - -- name (String) -- id (Uuid) -- route (String) -- method (String) -- status_code (I64) - -Imagine we wanted to look at the average latency by route, so averaged -across all methods and status codes. (Let's ingore name/id) - -We need to group the timeseries keys by route, to find the set of keys -consistent with each different route. ClickHouse provides the `groupArray` -function, which is an aggregate function that collects multiple values -into an array. So we can do: - -``` -SELECT - field_value, - groupArray(timeseries_key) -FROM fields_string -WHERE field_name = 'route' -GROUP BY field_value; - - -┌─field_value───────────────────────────────────────────┬─groupArray(timeseries_key)────────────────┐ -│ /metrics/producers │ [1916712826069192294,6228796576473532827] │ -│ /metrics/collectors │ [1500085842574282480] │ -│ /metrics/collect/e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c │ [15389669872422126367] │ -│ /sled_agents/fb0f7546-4d46-40ca-9d56-cbb810684ca7 │ [1166666993114742619] │ -└───────────────────────────────────────────────────────┴───────────────────────────────────────────┘ -``` - -This gives an array of timeseries keys where the route is each of the values -on the left. - -So at a very high level, we can average all the timeseries values where the keys -are in each of these different arrays. - - -This kinda works. It produces an array of arrays, the counts for each of the -histograms, grouped by the field value. - -``` -SELECT - field_value, - groupArray(counts) -FROM -( - SELECT - field_value, - timeseries_key - FROM fields_string - WHERE field_name = 'route' -) AS f0 -INNER JOIN -( - SELECT * - FROM measurements_histogramf64 -) AS meas USING (timeseries_key) -GROUP BY field_value -``` - -We can extend this `groupArray(bins), groupArray(counts)` to get both. - - -Ok, we're getting somewhere. The aggregation "combinators" modify the behavior of -aggregations, in pretty suprising and powerful ways. For example: - -``` -SELECT - field_value, - sumForEach(counts) -FROM -( - SELECT - field_value, - timeseries_key - FROM fields_string - WHERE field_name = 'route' -) AS f0 -INNER JOIN -( - SELECT * - FROM measurements_histogramf64 -) AS meas USING (timeseries_key) -GROUP BY field_value -``` - -This applies the `-ForEach` combinator to the sum aggregation. This applies the -aggregation to corresponding elements of a sequence (table?) of arrays. We can -do this with any of the aggregations, `avg`, `min`, etc. - - -The `-Resample` combinator also looks interesting. It uses its arguments to create -a set of intervals, and applies the aggregation within each of those intervals. -So sort of a group-by interval or window function. - -Another useful method is `toStartOfInterval`. This takes a timestamp and an interval, -say 5 seconds, or 10 minutes, and returns the interval into which that timestamp -falls. Could be very helpful for aligning/binning data to time intervals. But -it does "round", in that the bins don't start at the first timestamp, but at -the rounded-down interval from that timestamp. - -It's possible to build intervals that start exactly at the first timestamp with: - -``` -SELECT - timestamp, - toStartOfInterval(timestamp, toIntervalMinute(1)) + ( - SELECT toSecond(min(timestamp)) - FROM measurements_histogramf64 - ) -FROM measurements_histogramf64 -``` - -Or some other rounding shenanigans. - - -Putting lots of this together: - -``` -SELECT - f0.field_name, - f0.field_value, - f1.field_name, - f1.field_value, - minForEach(bins), - avgForEach(counts) -FROM -( - SELECT - field_name, - field_value, - timeseries_key - FROM fields_string - WHERE field_name = 'route' -) AS f0 -INNER JOIN -( - SELECT - field_name, - field_value, - timeseries_key - FROM fields_i64 - WHERE field_name = 'status_code' -) AS f1 ON f0.timeseries_key = f1.timeseries_key -INNER JOIN -( - SELECT * - FROM measurements_histogramf64 -) AS meas ON f1.timeseries_key = meas.timeseries_key -GROUP BY - f0.field_name, - f0.field_value, - f1.field_name, - f1.field_value -``` - -This selects the field name/value, and the bin and average count for each -histogram, grouping by route and status code. - -These inner select statements look similar to the ones we already -implement in `field.as_query`. But in that case we select *, and here we -probably don't want to do that to avoid errors about things not being -in aggregations or group by's. - -This works (or is syntactically valid) for scalars, if we replace the -combinators with their non-combinator version: e.g, `avgForEach` -> `avg`. - - -Other rando thoughts. - -It'd be nice to have the query builder be able to handle all these, but -I'm not sure how worth it that is. For example, I don't even think we need -the timeseries keys in this query. For the fields where we are specifying -a condition, we have subqueries like: - -``` -SELECT * -FROM fields_{TYPE} -WHERE field_name = NAME -AND field_value OP VALUE; -``` - -For ones where we _don't_ care, we just have the first three lines: - -``` -SELECT * -FROM fields_{TYPE} -WHERE field_name = NAME; -``` - -We can join successive entries on timeseries keys. - -For straight SELECT queries, that's pretty much it, like we have currently. -For AGGREGATION queries, we need to - -- Have a group-by for each (field_name, field_value) pair. This is true -even when we're unselective on the field, because we are still taking that -field, and we still need to group the keys accordingly. -- Select the consistent timeseries keys. This is so we can correlate the -results of the aggregation back to the field names/values which we still -get from the key-select query. -- Apply the aggregation to the measurements. For scalars, this just the -aggregation. For histograms, this is the `-Array` or `-ForEach` combinator -for that aggregation, depending on what we're applying. -- ??? to the timestamps? -- some alignment, grouping, subsampling? It seems -this has to come from the aggregation query, because there's not a useful -default. - -Speaking of defaults, how do these functions behave with missing data? -Or more subtly, what happens if two histograms (say) have the same number -of bins, but the actual bin edges are different? ClickHouse itself doesn't -deal with this AFAICT, which means we'd need to do that in the client. -Ah, but that is unlikely, since we're only aggregating data from the -same timeseries, with the same key. So far anyway. I'm not sure what'll -happen when we start correlating data between timeseries. diff --git a/oximeter/db/schema/replicated/4/up01.sql b/oximeter/db/schema/replicated/4/up01.sql new file mode 100644 index 0000000000..f36745ae2e --- /dev/null +++ b/oximeter/db/schema/replicated/4/up01.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_bool_local MODIFY COLUMN datum Nullable(UInt8) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up02.sql b/oximeter/db/schema/replicated/4/up02.sql new file mode 100644 index 0000000000..0f76398652 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up02.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_bool MODIFY COLUMN datum Nullable(UInt8) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up03.sql b/oximeter/db/schema/replicated/4/up03.sql new file mode 100644 index 0000000000..175b23d71b --- /dev/null +++ b/oximeter/db/schema/replicated/4/up03.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i8_local MODIFY COLUMN datum Nullable(Int8) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up04.sql b/oximeter/db/schema/replicated/4/up04.sql new file mode 100644 index 0000000000..4c8f22d8e6 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up04.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i8 MODIFY COLUMN datum Nullable(Int8) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up05.sql b/oximeter/db/schema/replicated/4/up05.sql new file mode 100644 index 0000000000..82490a81ca --- /dev/null +++ b/oximeter/db/schema/replicated/4/up05.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u8_local MODIFY COLUMN datum Nullable(UInt8) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up06.sql b/oximeter/db/schema/replicated/4/up06.sql new file mode 100644 index 0000000000..c689682127 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up06.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u8 MODIFY COLUMN datum Nullable(UInt8) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up07.sql b/oximeter/db/schema/replicated/4/up07.sql new file mode 100644 index 0000000000..43eb40515b --- /dev/null +++ b/oximeter/db/schema/replicated/4/up07.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i16_local MODIFY COLUMN datum Nullable(Int16) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up08.sql b/oximeter/db/schema/replicated/4/up08.sql new file mode 100644 index 0000000000..1d983a3c83 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up08.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i16 MODIFY COLUMN datum Nullable(Int16) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up09.sql b/oximeter/db/schema/replicated/4/up09.sql new file mode 100644 index 0000000000..e52c2adf5f --- /dev/null +++ b/oximeter/db/schema/replicated/4/up09.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u16_local MODIFY COLUMN datum Nullable(UInt16) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up10.sql b/oximeter/db/schema/replicated/4/up10.sql new file mode 100644 index 0000000000..d8a69fff1a --- /dev/null +++ b/oximeter/db/schema/replicated/4/up10.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u16 MODIFY COLUMN datum Nullable(UInt16) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up11.sql b/oximeter/db/schema/replicated/4/up11.sql new file mode 100644 index 0000000000..b3c2d8de92 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up11.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i32_local MODIFY COLUMN datum Nullable(Int32) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up12.sql b/oximeter/db/schema/replicated/4/up12.sql new file mode 100644 index 0000000000..65fca2e1b2 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up12.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i32 MODIFY COLUMN datum Nullable(Int32) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up13.sql b/oximeter/db/schema/replicated/4/up13.sql new file mode 100644 index 0000000000..df7c520e35 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up13.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u32_local MODIFY COLUMN datum Nullable(UInt32) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up14.sql b/oximeter/db/schema/replicated/4/up14.sql new file mode 100644 index 0000000000..a4cb43fb90 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up14.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u32 MODIFY COLUMN datum Nullable(UInt32) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up15.sql b/oximeter/db/schema/replicated/4/up15.sql new file mode 100644 index 0000000000..f7583dbdee --- /dev/null +++ b/oximeter/db/schema/replicated/4/up15.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i64_local MODIFY COLUMN datum Nullable(Int64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up16.sql b/oximeter/db/schema/replicated/4/up16.sql new file mode 100644 index 0000000000..b458243d74 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up16.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i64 MODIFY COLUMN datum Nullable(Int64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up17.sql b/oximeter/db/schema/replicated/4/up17.sql new file mode 100644 index 0000000000..9229a97704 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up17.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u64_local MODIFY COLUMN datum Nullable(UInt64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up18.sql b/oximeter/db/schema/replicated/4/up18.sql new file mode 100644 index 0000000000..6e2a2a5191 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up18.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u64 MODIFY COLUMN datum Nullable(UInt64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up19.sql b/oximeter/db/schema/replicated/4/up19.sql new file mode 100644 index 0000000000..8f16b5d41e --- /dev/null +++ b/oximeter/db/schema/replicated/4/up19.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_f32_local MODIFY COLUMN datum Nullable(Float32) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up20.sql b/oximeter/db/schema/replicated/4/up20.sql new file mode 100644 index 0000000000..9263592740 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up20.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_f32 MODIFY COLUMN datum Nullable(Float32) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up21.sql b/oximeter/db/schema/replicated/4/up21.sql new file mode 100644 index 0000000000..72abba6216 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up21.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_f64_local MODIFY COLUMN datum Nullable(Float64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up22.sql b/oximeter/db/schema/replicated/4/up22.sql new file mode 100644 index 0000000000..0d8522bc03 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up22.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_f64 MODIFY COLUMN datum Nullable(Float64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up23.sql b/oximeter/db/schema/replicated/4/up23.sql new file mode 100644 index 0000000000..96b94c2895 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up23.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativei64_local MODIFY COLUMN datum Nullable(Int64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up24.sql b/oximeter/db/schema/replicated/4/up24.sql new file mode 100644 index 0000000000..55df76c25f --- /dev/null +++ b/oximeter/db/schema/replicated/4/up24.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativei64 MODIFY COLUMN datum Nullable(Int64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up25.sql b/oximeter/db/schema/replicated/4/up25.sql new file mode 100644 index 0000000000..fac7369482 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up25.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativeu64_local MODIFY COLUMN datum Nullable(UInt64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up26.sql b/oximeter/db/schema/replicated/4/up26.sql new file mode 100644 index 0000000000..182b2b4704 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up26.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativeu64 MODIFY COLUMN datum Nullable(UInt64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up27.sql b/oximeter/db/schema/replicated/4/up27.sql new file mode 100644 index 0000000000..b482d00f81 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up27.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativef32_local MODIFY COLUMN datum Nullable(Float32) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up28.sql b/oximeter/db/schema/replicated/4/up28.sql new file mode 100644 index 0000000000..cefbe56395 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up28.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativef32 MODIFY COLUMN datum Nullable(Float32) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up29.sql b/oximeter/db/schema/replicated/4/up29.sql new file mode 100644 index 0000000000..59e21f353d --- /dev/null +++ b/oximeter/db/schema/replicated/4/up29.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativef64_local MODIFY COLUMN datum Nullable(Float64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up30.sql b/oximeter/db/schema/replicated/4/up30.sql new file mode 100644 index 0000000000..a609e6ad3c --- /dev/null +++ b/oximeter/db/schema/replicated/4/up30.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativef64 MODIFY COLUMN datum Nullable(Float64) \ No newline at end of file diff --git a/oximeter/db/schema/replicated/4/up31.sql b/oximeter/db/schema/replicated/4/up31.sql new file mode 100644 index 0000000000..3726895dd0 --- /dev/null +++ b/oximeter/db/schema/replicated/4/up31.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_string_local MODIFY COLUMN datum Nullable(String); diff --git a/oximeter/db/schema/replicated/4/up32.sql b/oximeter/db/schema/replicated/4/up32.sql new file mode 100644 index 0000000000..5a09705e7e --- /dev/null +++ b/oximeter/db/schema/replicated/4/up32.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_string MODIFY COLUMN datum Nullable(String); diff --git a/oximeter/db/schema/replicated/db-init.sql b/oximeter/db/schema/replicated/db-init.sql index 4429f41364..27df02b709 100644 --- a/oximeter/db/schema/replicated/db-init.sql +++ b/oximeter/db/schema/replicated/db-init.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool_local ON CLUSTER oximeter_ timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt8 + datum Nullable(UInt8) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_bool_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool ON CLUSTER oximeter_cluste timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt8 + datum Nullable(UInt8) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_bool_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -44,7 +44,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i8_local ON CLUSTER oximeter_cl timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int8 + datum Nullable(Int8) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i8_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -55,7 +55,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i8 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int8 + datum Nullable(Int8) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i8_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u8_local ON CLUSTER oximeter_cl timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt8 + datum Nullable(UInt8) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u8_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -75,7 +75,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u8 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt8 + datum Nullable(UInt8) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u8_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -84,7 +84,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i16_local ON CLUSTER oximeter_c timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int16 + datum Nullable(Int16) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i16_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -95,7 +95,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i16 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int16 + datum Nullable(Int16) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i16_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -104,7 +104,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u16_local ON CLUSTER oximeter_c timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt16 + datum Nullable(UInt16) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u16_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u16 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt16 + datum Nullable(UInt16) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u16_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -124,7 +124,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i32_local ON CLUSTER oximeter_c timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int32 + datum Nullable(Int32) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -135,7 +135,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i32 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int32 + datum Nullable(Int32) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i32_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -144,7 +144,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u32_local ON CLUSTER oximeter_c timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt32 + datum Nullable(UInt32) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -155,7 +155,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u32 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt32 + datum Nullable(UInt32) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u32_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -164,7 +164,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i64_local ON CLUSTER oximeter_c timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int64 + datum Nullable(Int64) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -175,7 +175,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i64 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int64 + datum Nullable(Int64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i64_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -184,7 +184,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u64_local ON CLUSTER oximeter_c timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt64 + datum Nullable(UInt64) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -195,7 +195,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u64 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt64 + datum Nullable(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u64_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -204,7 +204,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f32_local ON CLUSTER oximeter_c timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Float32 + datum Nullable(Float32) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_f32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -215,7 +215,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f32 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Float32 + datum Nullable(Float32) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_f32_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -224,7 +224,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f64_local ON CLUSTER oximeter_c timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Float64 + datum Nullable(Float64) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_f64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -235,7 +235,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f64 ON CLUSTER oximeter_cluster timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Float64 + datum Nullable(Float64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_f64_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -244,7 +244,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_string_local ON CLUSTER oximete timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum String + datum Nullable(String) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_string_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -255,7 +255,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_string ON CLUSTER oximeter_clus timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum String + datum Nullable(String) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_string_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -285,7 +285,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64_local ON CLUSTER timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum Int64 + datum Nullable(Int64) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativei64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) @@ -297,7 +297,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64 ON CLUSTER oximet timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum Int64 + datum Nullable(Int64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativei64_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -307,7 +307,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64_local ON CLUSTER timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum UInt64 + datum Nullable(UInt64) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativeu64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) @@ -319,7 +319,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64 ON CLUSTER oximet timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum UInt64 + datum Nullable(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativeu64_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -329,7 +329,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32_local ON CLUSTER timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum Float32 + datum Nullable(Float32) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativef32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) @@ -341,7 +341,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32 ON CLUSTER oximet timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum Float32 + datum Nullable(Float32) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativef32_local', xxHash64(splitByChar(':', timeseries_name)[1])); @@ -351,7 +351,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64_local ON CLUSTER timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum Float64 + datum Nullable(Float64) ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativef64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) @@ -363,7 +363,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64 ON CLUSTER oximet timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum Float64 + datum Nullable(Float64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativef64_local', xxHash64(splitByChar(':', timeseries_name)[1])); diff --git a/oximeter/db/schema/single-node/4/up01.sql b/oximeter/db/schema/single-node/4/up01.sql new file mode 100644 index 0000000000..ccccc9c5fb --- /dev/null +++ b/oximeter/db/schema/single-node/4/up01.sql @@ -0,0 +1,9 @@ +/* + * To support missing measurements, we are making all scalar datum columns + * Nullable, so that a NULL value (None in Rust) represents a missing datum at + * the provided timestamp. + * + * Note that arrays cannot be made Nullable, so we need to use an empty array as + * the sentinel value implying a missing measurement. + */ +ALTER TABLE oximeter.measurements_bool MODIFY COLUMN datum Nullable(UInt8) diff --git a/oximeter/db/schema/single-node/4/up02.sql b/oximeter/db/schema/single-node/4/up02.sql new file mode 100644 index 0000000000..4c8f22d8e6 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up02.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i8 MODIFY COLUMN datum Nullable(Int8) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up03.sql b/oximeter/db/schema/single-node/4/up03.sql new file mode 100644 index 0000000000..c689682127 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up03.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u8 MODIFY COLUMN datum Nullable(UInt8) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up04.sql b/oximeter/db/schema/single-node/4/up04.sql new file mode 100644 index 0000000000..1d983a3c83 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up04.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i16 MODIFY COLUMN datum Nullable(Int16) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up05.sql b/oximeter/db/schema/single-node/4/up05.sql new file mode 100644 index 0000000000..d8a69fff1a --- /dev/null +++ b/oximeter/db/schema/single-node/4/up05.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u16 MODIFY COLUMN datum Nullable(UInt16) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up06.sql b/oximeter/db/schema/single-node/4/up06.sql new file mode 100644 index 0000000000..65fca2e1b2 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up06.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i32 MODIFY COLUMN datum Nullable(Int32) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up07.sql b/oximeter/db/schema/single-node/4/up07.sql new file mode 100644 index 0000000000..a4cb43fb90 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up07.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u32 MODIFY COLUMN datum Nullable(UInt32) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up08.sql b/oximeter/db/schema/single-node/4/up08.sql new file mode 100644 index 0000000000..b458243d74 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up08.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_i64 MODIFY COLUMN datum Nullable(Int64) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up09.sql b/oximeter/db/schema/single-node/4/up09.sql new file mode 100644 index 0000000000..6e2a2a5191 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up09.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_u64 MODIFY COLUMN datum Nullable(UInt64) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up10.sql b/oximeter/db/schema/single-node/4/up10.sql new file mode 100644 index 0000000000..9263592740 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up10.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_f32 MODIFY COLUMN datum Nullable(Float32) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up11.sql b/oximeter/db/schema/single-node/4/up11.sql new file mode 100644 index 0000000000..0d8522bc03 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up11.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_f64 MODIFY COLUMN datum Nullable(Float64) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up12.sql b/oximeter/db/schema/single-node/4/up12.sql new file mode 100644 index 0000000000..55df76c25f --- /dev/null +++ b/oximeter/db/schema/single-node/4/up12.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativei64 MODIFY COLUMN datum Nullable(Int64) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up13.sql b/oximeter/db/schema/single-node/4/up13.sql new file mode 100644 index 0000000000..182b2b4704 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up13.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativeu64 MODIFY COLUMN datum Nullable(UInt64) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up14.sql b/oximeter/db/schema/single-node/4/up14.sql new file mode 100644 index 0000000000..cefbe56395 --- /dev/null +++ b/oximeter/db/schema/single-node/4/up14.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativef32 MODIFY COLUMN datum Nullable(Float32) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up15.sql b/oximeter/db/schema/single-node/4/up15.sql new file mode 100644 index 0000000000..a609e6ad3c --- /dev/null +++ b/oximeter/db/schema/single-node/4/up15.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_cumulativef64 MODIFY COLUMN datum Nullable(Float64) \ No newline at end of file diff --git a/oximeter/db/schema/single-node/4/up16.sql b/oximeter/db/schema/single-node/4/up16.sql new file mode 100644 index 0000000000..5a09705e7e --- /dev/null +++ b/oximeter/db/schema/single-node/4/up16.sql @@ -0,0 +1 @@ +ALTER TABLE oximeter.measurements_string MODIFY COLUMN datum Nullable(String); diff --git a/oximeter/db/schema/single-node/db-init.sql b/oximeter/db/schema/single-node/db-init.sql index ee5e91c4b7..510c1071c8 100644 --- a/oximeter/db/schema/single-node/db-init.sql +++ b/oximeter/db/schema/single-node/db-init.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt8 + datum Nullable(UInt8) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i8 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int8 + datum Nullable(Int8) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -46,7 +46,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u8 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt8 + datum Nullable(UInt8) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -57,7 +57,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i16 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int16 + datum Nullable(Int16) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -68,7 +68,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u16 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt16 + datum Nullable(UInt16) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -79,7 +79,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i32 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int32 + datum Nullable(Int32) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -90,7 +90,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u32 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt32 + datum Nullable(UInt32) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -101,7 +101,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i64 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Int64 + datum Nullable(Int64) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -112,7 +112,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u64 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum UInt64 + datum Nullable(UInt64) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -123,7 +123,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f32 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Float32 + datum Nullable(Float32) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -134,7 +134,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f64 timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum Float64 + datum Nullable(Float64) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -145,7 +145,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_string timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), - datum String + datum Nullable(String) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) @@ -156,6 +156,13 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes timeseries_name String, timeseries_key UInt64, timestamp DateTime64(9, 'UTC'), + /* + * NOTE: Right now we can't unambiguously record a nullable byte array. + * Arrays cannot be nested in `Nullable()` types, and encoding the array as + * a string isn't palatable for a few reasons. + * See: https://github.com/oxidecomputer/omicron/issues/4551 for more + * details. + */ datum Array(UInt8) ) ENGINE = MergeTree() @@ -168,7 +175,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64 timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum Int64 + datum Nullable(Int64) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) @@ -180,7 +187,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64 timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum UInt64 + datum Nullable(UInt64) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) @@ -192,7 +199,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32 timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum Float32 + datum Nullable(Float32) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) @@ -205,7 +212,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64 timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), - datum Float64 + datum Nullable(Float64) ) ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) @@ -217,6 +224,16 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8 timeseries_key UInt64, start_time DateTime64(9, 'UTC'), timestamp DateTime64(9, 'UTC'), + /* + * NOTE: Array types cannot be Nullable, see + * https://clickhouse.com/docs/en/sql-reference/data-types/nullable + * for more details. + * + * This means we need to use empty arrays to indicate a missing value. This + * is unfortunate, and at this point relies on the fact that an + * `oximeter::Histogram` cannot have zero bins. If that changes, we'll need + * to figure out another way to represent missing samples here. + */ bins Array(Int8), counts Array(UInt64) ) diff --git a/oximeter/db/src/client.rs b/oximeter/db/src/client.rs index e1ed06554c..c8a7db20cb 100644 --- a/oximeter/db/src/client.rs +++ b/oximeter/db/src/client.rs @@ -1190,7 +1190,7 @@ mod tests { use super::*; use crate::query; use crate::query::field_table_name; - use crate::query::measurement_table_name; + use bytes::Bytes; use chrono::Utc; use omicron_test_utils::dev::clickhouse::{ ClickHouseCluster, ClickHouseInstance, @@ -1198,8 +1198,10 @@ mod tests { use omicron_test_utils::dev::test_setup_log; use oximeter::histogram::Histogram; use oximeter::test_util; + use oximeter::types::MissingDatum; use oximeter::Datum; use oximeter::FieldValue; + use oximeter::Measurement; use oximeter::Metric; use oximeter::Target; use std::net::Ipv6Addr; @@ -2957,76 +2959,102 @@ mod tests { Ok(()) } + async fn test_recall_missing_scalar_measurement_impl( + measurement: Measurement, + client: &Client, + ) -> Result<(), Error> { + let start_time = if measurement.datum().is_cumulative() { + Some(Utc::now()) + } else { + None + }; + let missing_datum = Datum::from( + MissingDatum::new(measurement.datum_type(), start_time).unwrap(), + ); + let missing_measurement = Measurement::new(Utc::now(), missing_datum); + test_recall_measurement_impl(missing_measurement, client).await?; + Ok(()) + } + async fn recall_measurement_bool_test( client: &Client, ) -> Result<(), Error> { let datum = Datum::Bool(true); - let as_json = serde_json::Value::from(1_u64); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } async fn recall_measurement_i8_test(client: &Client) -> Result<(), Error> { let datum = Datum::I8(1); - let as_json = serde_json::Value::from(1_i8); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } async fn recall_measurement_u8_test(client: &Client) -> Result<(), Error> { let datum = Datum::U8(1); - let as_json = serde_json::Value::from(1_u8); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } async fn recall_measurement_i16_test(client: &Client) -> Result<(), Error> { let datum = Datum::I16(1); - let as_json = serde_json::Value::from(1_i16); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } async fn recall_measurement_u16_test(client: &Client) -> Result<(), Error> { let datum = Datum::U16(1); - let as_json = serde_json::Value::from(1_u16); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } async fn recall_measurement_i32_test(client: &Client) -> Result<(), Error> { let datum = Datum::I32(1); - let as_json = serde_json::Value::from(1_i32); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } async fn recall_measurement_u32_test(client: &Client) -> Result<(), Error> { let datum = Datum::U32(1); - let as_json = serde_json::Value::from(1_u32); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } async fn recall_measurement_i64_test(client: &Client) -> Result<(), Error> { let datum = Datum::I64(1); - let as_json = serde_json::Value::from(1_i64); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } async fn recall_measurement_u64_test(client: &Client) -> Result<(), Error> { let datum = Datum::U64(1); - let as_json = serde_json::Value::from(1_u64); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } @@ -3034,9 +3062,9 @@ mod tests { async fn recall_measurement_f32_test(client: &Client) -> Result<(), Error> { const VALUE: f32 = 1.1; let datum = Datum::F32(VALUE); - // NOTE: This is intentionally an f64. - let as_json = serde_json::Value::from(1.1_f64); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } @@ -3044,18 +3072,43 @@ mod tests { async fn recall_measurement_f64_test(client: &Client) -> Result<(), Error> { const VALUE: f64 = 1.1; let datum = Datum::F64(VALUE); - let as_json = serde_json::Value::from(VALUE); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } + async fn recall_measurement_string_test( + client: &Client, + ) -> Result<(), Error> { + let value = String::from("foo"); + let datum = Datum::String(value.clone()); + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) + .await?; + Ok(()) + } + + async fn recall_measurement_bytes_test( + client: &Client, + ) -> Result<(), Error> { + let value = Bytes::from(vec![0, 1, 2]); + let datum = Datum::Bytes(value.clone()); + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + // NOTE: We don't currently support missing byte array samples. + Ok(()) + } + async fn recall_measurement_cumulative_i64_test( client: &Client, ) -> Result<(), Error> { let datum = Datum::CumulativeI64(1.into()); - let as_json = serde_json::Value::from(1_i64); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } @@ -3064,8 +3117,9 @@ mod tests { client: &Client, ) -> Result<(), Error> { let datum = Datum::CumulativeU64(1.into()); - let as_json = serde_json::Value::from(1_u64); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } @@ -3074,8 +3128,9 @@ mod tests { client: &Client, ) -> Result<(), Error> { let datum = Datum::CumulativeF64(1.1.into()); - let as_json = serde_json::Value::from(1.1_f64); - test_recall_measurement_impl::(datum, None, as_json, client) + let measurement = Measurement::new(Utc::now(), datum); + test_recall_measurement_impl(measurement.clone(), client).await?; + test_recall_missing_scalar_measurement_impl(measurement, client) .await?; Ok(()) } @@ -3089,13 +3144,15 @@ mod tests { Datum: From>, serde_json::Value: From, { - let (bins, counts) = hist.to_arrays(); let datum = Datum::from(hist); - let as_json = serde_json::Value::Array( - counts.into_iter().map(Into::into).collect(), + let measurement = Measurement::new(Utc::now(), datum); + let missing_datum = Datum::Missing( + MissingDatum::new(measurement.datum_type(), Some(Utc::now())) + .unwrap(), ); - test_recall_measurement_impl(datum, Some(bins), as_json, client) - .await?; + let missing_measurement = Measurement::new(Utc::now(), missing_datum); + test_recall_measurement_impl(measurement, client).await?; + test_recall_measurement_impl(missing_measurement, client).await?; Ok(()) } @@ -3192,54 +3249,23 @@ mod tests { Ok(()) } - async fn test_recall_measurement_impl + Copy>( - datum: Datum, - maybe_bins: Option>, - json_datum: serde_json::Value, + async fn test_recall_measurement_impl( + measurement: Measurement, client: &Client, ) -> Result<(), Error> { // Insert a record from this datum. const TIMESERIES_NAME: &str = "foo:bar"; const TIMESERIES_KEY: u64 = 101; - let mut inserted_row = serde_json::Map::new(); - inserted_row - .insert("timeseries_name".to_string(), TIMESERIES_NAME.into()); - inserted_row - .insert("timeseries_key".to_string(), TIMESERIES_KEY.into()); - inserted_row.insert( - "timestamp".to_string(), - Utc::now() - .format(crate::DATABASE_TIMESTAMP_FORMAT) - .to_string() - .into(), - ); - - // Insert the start time and possibly bins. - if let Some(start_time) = datum.start_time() { - inserted_row.insert( - "start_time".to_string(), - start_time - .format(crate::DATABASE_TIMESTAMP_FORMAT) - .to_string() - .into(), - ); - } - if let Some(bins) = &maybe_bins { - let bins = serde_json::Value::Array( - bins.iter().copied().map(Into::into).collect(), + let (measurement_table, inserted_row) = + crate::model::unroll_measurement_row_impl( + TIMESERIES_NAME.to_string(), + TIMESERIES_KEY, + &measurement, ); - inserted_row.insert("bins".to_string(), bins); - inserted_row.insert("counts".to_string(), json_datum); - } else { - inserted_row.insert("datum".to_string(), json_datum); - } - let inserted_row = serde_json::Value::from(inserted_row); - - let measurement_table = measurement_table_name(datum.datum_type()); - let row = serde_json::to_string(&inserted_row).unwrap(); let insert_sql = format!( - "INSERT INTO oximeter.{measurement_table} FORMAT JSONEachRow {row}", + "INSERT INTO {measurement_table} FORMAT JSONEachRow {inserted_row}", ); + println!("Inserted row: {}", inserted_row); client .execute(insert_sql) .await @@ -3247,21 +3273,22 @@ mod tests { // Select it exactly back out. let select_sql = format!( - "SELECT * FROM oximeter.{} LIMIT 2 FORMAT {};", + "SELECT * FROM {} WHERE timestamp = '{}' FORMAT {};", measurement_table, + measurement.timestamp().format(crate::DATABASE_TIMESTAMP_FORMAT), crate::DATABASE_SELECT_FORMAT, ); let body = client .execute_with_body(select_sql) .await .expect("Failed to select measurement row"); - println!("{}", body); - let actual_row: serde_json::Value = serde_json::from_str(&body) - .expect("Failed to parse measurement row JSON"); - println!("{actual_row:?}"); - println!("{inserted_row:?}"); + let (_, actual_row) = crate::model::parse_measurement_from_row( + &body, + measurement.datum_type(), + ); + println!("Actual row: {actual_row:?}"); assert_eq!( - actual_row, inserted_row, + actual_row, measurement, "Actual and expected measurement rows do not match" ); Ok(()) @@ -3311,6 +3338,10 @@ mod tests { recall_measurement_f64_test(&client).await.unwrap(); + recall_measurement_string_test(&client).await.unwrap(); + + recall_measurement_bytes_test(&client).await.unwrap(); + recall_measurement_cumulative_i64_test(&client).await.unwrap(); recall_measurement_cumulative_u64_test(&client).await.unwrap(); diff --git a/oximeter/db/src/model.rs b/oximeter/db/src/model.rs index 715e025a04..d92e646e89 100644 --- a/oximeter/db/src/model.rs +++ b/oximeter/db/src/model.rs @@ -26,6 +26,7 @@ use oximeter::types::Field; use oximeter::types::FieldType; use oximeter::types::FieldValue; use oximeter::types::Measurement; +use oximeter::types::MissingDatum; use oximeter::types::Sample; use serde::Deserialize; use serde::Serialize; @@ -43,7 +44,7 @@ use uuid::Uuid; /// - [`crate::Client::initialize_db_with_version`] /// - [`crate::Client::ensure_schema`] /// - The `clickhouse-schema-updater` binary in this crate -pub const OXIMETER_VERSION: u64 = 3; +pub const OXIMETER_VERSION: u64 = 4; // Wrapper type to represent a boolean in the database. // @@ -212,6 +213,7 @@ impl From for DbFieldType { } } } + #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)] pub enum DbDatumType { Bool, @@ -402,7 +404,7 @@ macro_rules! declare_measurement_row { timeseries_key: TimeseriesKey, #[serde(with = "serde_timestamp")] timestamp: DateTime, - datum: $datum_type, + datum: Option<$datum_type>, } impl_table_name!{$name, "measurements", $data_type} @@ -433,7 +435,7 @@ macro_rules! declare_cumulative_measurement_row { start_time: DateTime, #[serde(with = "serde_timestamp")] timestamp: DateTime, - datum: $datum_type, + datum: Option<$datum_type>, } impl_table_name!{$name, "measurements", $data_type} @@ -456,6 +458,22 @@ struct DbHistogram { pub counts: Vec, } +// We use an empty histogram to indicate a missing sample. +// +// While ClickHouse supports nullable types, the inner type can't be a +// "composite", which includes arrays. I.e., `Nullable(Array(UInt8))` can't be +// used. This is unfortunate, but we are aided by the fact that it's not +// possible to have an `oximeter` histogram that contains zero bins right now. +// This is checked by a test in `oximeter::histogram`. +// +// That means we can currently use an empty array from the database as a +// sentinel for a missing sample. +impl DbHistogram { + fn null() -> Self { + Self { bins: vec![], counts: vec![] } + } +} + impl From<&Histogram> for DbHistogram where T: traits::HistogramSupport, @@ -647,270 +665,571 @@ pub(crate) fn unroll_measurement_row(sample: &Sample) -> (String, String) { let timeseries_name = sample.timeseries_name.clone(); let timeseries_key = crate::timeseries_key(sample); let measurement = &sample.measurement; + unroll_measurement_row_impl(timeseries_name, timeseries_key, measurement) +} + +/// Given a sample's measurement, return a table name and row to insert. +/// +/// This returns a tuple giving the name of the table, and the JSON +/// representation for the serialized row to be inserted into that table, +/// written out as a string. +pub(crate) fn unroll_measurement_row_impl( + timeseries_name: String, + timeseries_key: TimeseriesKey, + measurement: &Measurement, +) -> (String, String) { let timestamp = measurement.timestamp(); let extract_start_time = |measurement: &Measurement| { measurement .start_time() .expect("Cumulative measurements must have a start time") }; + match measurement.datum() { Datum::Bool(inner) => { + let datum = Some(DbBool::from(*inner)); let row = BoolMeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: DbBool::from(*inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::I8(inner) => { + let datum = Some(*inner); let row = I8MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::U8(inner) => { + let datum = Some(*inner); let row = U8MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::I16(inner) => { + let datum = Some(*inner); let row = I16MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::U16(inner) => { + let datum = Some(*inner); let row = U16MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::I32(inner) => { + let datum = Some(*inner); let row = I32MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::U32(inner) => { + let datum = Some(*inner); let row = U32MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::I64(inner) => { + let datum = Some(*inner); let row = I64MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::U64(inner) => { + let datum = Some(*inner); let row = U64MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::F32(inner) => { + let datum = Some(*inner); let row = F32MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::F64(inner) => { + let datum = Some(*inner); let row = F64MeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: *inner, + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::String(ref inner) => { + Datum::String(inner) => { + let datum = Some(inner.clone()); let row = StringMeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: inner.clone(), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::Bytes(ref inner) => { + Datum::Bytes(inner) => { + let datum = Some(inner.clone()); let row = BytesMeasurementRow { timeseries_name, timeseries_key, timestamp, - datum: inner.clone(), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::CumulativeI64(inner) => { + let datum = Some(inner.value()); let row = CumulativeI64MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: inner.value(), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::CumulativeU64(inner) => { + let datum = Some(inner.value()); let row = CumulativeU64MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: inner.value(), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::CumulativeF32(inner) => { + let datum = Some(inner.value()); let row = CumulativeF32MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: inner.value(), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } Datum::CumulativeF64(inner) => { + let datum = Some(inner.value()); let row = CumulativeF64MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: inner.value(), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramI8(ref inner) => { + Datum::HistogramI8(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramI8MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramU8(ref inner) => { + Datum::HistogramU8(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramU8MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramI16(ref inner) => { + Datum::HistogramI16(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramI16MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramU16(ref inner) => { + Datum::HistogramU16(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramU16MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramI32(ref inner) => { + Datum::HistogramI32(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramI32MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramU32(ref inner) => { + Datum::HistogramU32(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramU32MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramI64(ref inner) => { + Datum::HistogramI64(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramI64MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramU64(ref inner) => { + Datum::HistogramU64(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramU64MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramF32(ref inner) => { + Datum::HistogramF32(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramF32MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } - Datum::HistogramF64(ref inner) => { + Datum::HistogramF64(inner) => { + let datum = DbHistogram::from(inner); let row = HistogramF64MeasurementRow { timeseries_name, timeseries_key, start_time: extract_start_time(measurement), timestamp, - datum: DbHistogram::from(inner), + datum, }; (row.table_name(), serde_json::to_string(&row).unwrap()) } + Datum::Missing(missing) => { + match missing.datum_type() { + DatumType::Bool => { + let row = BoolMeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::I8 => { + let row = I8MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::U8 => { + let row = U8MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::I16 => { + let row = I16MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::U16 => { + let row = U16MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::I32 => { + let row = I32MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::U32 => { + let row = U32MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::I64 => { + let row = I64MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::U64 => { + let row = U64MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::F32 => { + let row = F32MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::F64 => { + let row = F64MeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::String => { + let row = StringMeasurementRow { + timeseries_name, + timeseries_key, + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::Bytes => { + // See https://github.com/oxidecomputer/omicron/issues/4551. + // + // This is actually unreachable today because the constuctor + // for `oximeter::types::MissingDatum` fails when using a + // `DatumType::Bytes`. + unreachable!(); + } + DatumType::CumulativeI64 => { + let row = CumulativeI64MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::CumulativeU64 => { + let row = CumulativeU64MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::CumulativeF32 => { + let row = CumulativeF32MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::CumulativeF64 => { + let row = CumulativeF64MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: None, + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramI8 => { + let row = HistogramI8MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramU8 => { + let row = HistogramU8MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramI16 => { + let row = HistogramI16MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramU16 => { + let row = HistogramU16MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramI32 => { + let row = HistogramI32MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramU32 => { + let row = HistogramU32MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramI64 => { + let row = HistogramI64MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramU64 => { + let row = HistogramU64MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramF32 => { + let row = HistogramF32MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + DatumType::HistogramF64 => { + let row = HistogramF64MeasurementRow { + timeseries_name, + timeseries_key, + start_time: extract_start_time(measurement), + timestamp, + datum: DbHistogram::null(), + }; + (row.table_name(), serde_json::to_string(&row).unwrap()) + } + } + } } } @@ -984,7 +1303,7 @@ struct DbTimeseriesScalarGaugeSample { timeseries_key: TimeseriesKey, #[serde(with = "serde_timestamp")] timestamp: DateTime, - datum: T, + datum: Option, } // A scalar timestamped sample from a cumulative timeseries, as extracted from a query to the @@ -996,7 +1315,7 @@ struct DbTimeseriesScalarCumulativeSample { start_time: DateTime, #[serde(with = "serde_timestamp")] timestamp: DateTime, - datum: T, + datum: Option, } // A histogram timestamped sample from a timeseries, as extracted from a query to the database. @@ -1014,9 +1333,15 @@ struct DbTimeseriesHistogramSample { impl From> for Measurement where Datum: From, + T: FromDbScalar, { fn from(sample: DbTimeseriesScalarGaugeSample) -> Measurement { - let datum = Datum::from(sample.datum); + let datum = match sample.datum { + Some(datum) => Datum::from(datum), + None => { + Datum::Missing(MissingDatum::new(T::DATUM_TYPE, None).unwrap()) + } + }; Measurement::new(sample.timestamp, datum) } } @@ -1024,12 +1349,19 @@ where impl From> for Measurement where Datum: From>, - T: traits::Cumulative, + T: traits::Cumulative + FromDbCumulative, { fn from(sample: DbTimeseriesScalarCumulativeSample) -> Measurement { - let cumulative = - Cumulative::with_start_time(sample.start_time, sample.datum); - let datum = Datum::from(cumulative); + let datum = match sample.datum { + Some(datum) => Datum::from(Cumulative::with_start_time( + sample.start_time, + datum, + )), + None => Datum::Missing( + MissingDatum::new(T::DATUM_TYPE, Some(sample.start_time)) + .unwrap(), + ), + }; Measurement::new(sample.timestamp, datum) } } @@ -1037,26 +1369,157 @@ where impl From> for Measurement where Datum: From>, - T: traits::HistogramSupport, + T: traits::HistogramSupport + FromDbHistogram, { fn from(sample: DbTimeseriesHistogramSample) -> Measurement { - let datum = Datum::from( - Histogram::from_arrays( - sample.start_time, - sample.bins, - sample.counts, + let datum = if sample.bins.is_empty() { + assert!(sample.counts.is_empty()); + Datum::Missing( + MissingDatum::new(T::DATUM_TYPE, Some(sample.start_time)) + .unwrap(), ) - .unwrap(), - ); + } else { + Datum::from( + Histogram::from_arrays( + sample.start_time, + sample.bins, + sample.counts, + ) + .unwrap(), + ) + }; Measurement::new(sample.timestamp, datum) } } +// Helper trait providing the DatumType for a corresponding scalar DB value. +// +// This is used in `parse_timeseries_scalar_gauge_measurement`. +trait FromDbScalar { + const DATUM_TYPE: DatumType; +} + +impl FromDbScalar for DbBool { + const DATUM_TYPE: DatumType = DatumType::Bool; +} + +impl FromDbScalar for i8 { + const DATUM_TYPE: DatumType = DatumType::I8; +} + +impl FromDbScalar for u8 { + const DATUM_TYPE: DatumType = DatumType::U8; +} + +impl FromDbScalar for i16 { + const DATUM_TYPE: DatumType = DatumType::I16; +} + +impl FromDbScalar for u16 { + const DATUM_TYPE: DatumType = DatumType::U16; +} + +impl FromDbScalar for i32 { + const DATUM_TYPE: DatumType = DatumType::I32; +} + +impl FromDbScalar for u32 { + const DATUM_TYPE: DatumType = DatumType::U32; +} + +impl FromDbScalar for i64 { + const DATUM_TYPE: DatumType = DatumType::I64; +} + +impl FromDbScalar for u64 { + const DATUM_TYPE: DatumType = DatumType::U64; +} + +impl FromDbScalar for f32 { + const DATUM_TYPE: DatumType = DatumType::F32; +} + +impl FromDbScalar for f64 { + const DATUM_TYPE: DatumType = DatumType::F64; +} + +impl FromDbScalar for String { + const DATUM_TYPE: DatumType = DatumType::String; +} + +impl FromDbScalar for Bytes { + const DATUM_TYPE: DatumType = DatumType::Bytes; +} + +trait FromDbCumulative { + const DATUM_TYPE: DatumType; +} + +impl FromDbCumulative for i64 { + const DATUM_TYPE: DatumType = DatumType::CumulativeI64; +} + +impl FromDbCumulative for u64 { + const DATUM_TYPE: DatumType = DatumType::CumulativeU64; +} + +impl FromDbCumulative for f32 { + const DATUM_TYPE: DatumType = DatumType::CumulativeF32; +} + +impl FromDbCumulative for f64 { + const DATUM_TYPE: DatumType = DatumType::CumulativeF64; +} + +trait FromDbHistogram { + const DATUM_TYPE: DatumType; +} + +impl FromDbHistogram for i8 { + const DATUM_TYPE: DatumType = DatumType::HistogramI8; +} + +impl FromDbHistogram for u8 { + const DATUM_TYPE: DatumType = DatumType::HistogramU8; +} + +impl FromDbHistogram for i16 { + const DATUM_TYPE: DatumType = DatumType::HistogramI16; +} + +impl FromDbHistogram for u16 { + const DATUM_TYPE: DatumType = DatumType::HistogramU16; +} + +impl FromDbHistogram for i32 { + const DATUM_TYPE: DatumType = DatumType::HistogramI32; +} + +impl FromDbHistogram for u32 { + const DATUM_TYPE: DatumType = DatumType::HistogramU32; +} + +impl FromDbHistogram for i64 { + const DATUM_TYPE: DatumType = DatumType::HistogramI64; +} + +impl FromDbHistogram for u64 { + const DATUM_TYPE: DatumType = DatumType::HistogramU64; +} + +impl FromDbHistogram for f32 { + const DATUM_TYPE: DatumType = DatumType::HistogramF32; +} + +impl FromDbHistogram for f64 { + const DATUM_TYPE: DatumType = DatumType::HistogramF64; +} + fn parse_timeseries_scalar_gauge_measurement<'a, T>( line: &'a str, ) -> (TimeseriesKey, Measurement) where - T: Deserialize<'a> + Into, + T: Deserialize<'a> + Into + FromDbScalar, Datum: From, { let sample = @@ -1068,7 +1531,7 @@ fn parse_timeseries_scalar_cumulative_measurement<'a, T>( line: &'a str, ) -> (TimeseriesKey, Measurement) where - T: Deserialize<'a> + traits::Cumulative, + T: Deserialize<'a> + traits::Cumulative + FromDbCumulative, Datum: From>, { let sample = @@ -1081,7 +1544,7 @@ fn parse_timeseries_histogram_measurement( line: &str, ) -> (TimeseriesKey, Measurement) where - T: Into + traits::HistogramSupport, + T: Into + traits::HistogramSupport + FromDbHistogram, Datum: From>, { let sample = @@ -1459,6 +1922,27 @@ mod tests { } } + // Test that we correctly unroll a row when the measurement is missing its + // datum. + #[test] + fn test_unroll_missing_measurement_row() { + let sample = test_util::make_sample(); + let missing_sample = test_util::make_missing_sample(); + let (table_name, row) = unroll_measurement_row(&sample); + let (missing_table_name, missing_row) = + unroll_measurement_row(&missing_sample); + let row = serde_json::from_str::(&row).unwrap(); + let missing_row = + serde_json::from_str::(&missing_row).unwrap(); + println!("{row:#?}"); + println!("{missing_row:#?}"); + assert_eq!(table_name, missing_table_name); + assert_eq!(row.timeseries_name, missing_row.timeseries_name); + assert_eq!(row.timeseries_key, missing_row.timeseries_key); + assert!(row.datum.is_some()); + assert!(missing_row.datum.is_none()); + } + #[test] fn test_unroll_measurement_row() { let sample = test_util::make_hist_sample(); @@ -1473,14 +1957,13 @@ mod tests { ) .unwrap(); let measurement = &sample.measurement; - if let Datum::HistogramF64(hist) = measurement.datum() { - assert_eq!( - hist, &unpacked_hist, - "Unpacking histogram from database representation failed" - ); - } else { + let Datum::HistogramF64(hist) = measurement.datum() else { panic!("Expected a histogram measurement"); - } + }; + assert_eq!( + hist, &unpacked_hist, + "Unpacking histogram from database representation failed" + ); assert_eq!(unpacked.start_time, measurement.start_time().unwrap()); } @@ -1582,12 +2065,11 @@ mod tests { assert_eq!(key, 12); assert_eq!(measurement.start_time().unwrap(), start_time); assert_eq!(measurement.timestamp(), timestamp); - if let Datum::HistogramI64(hist) = measurement.datum() { - assert_eq!(hist.n_bins(), 3); - assert_eq!(hist.n_samples(), 2); - } else { + let Datum::HistogramI64(hist) = measurement.datum() else { panic!("Expected a histogram sample"); - } + }; + assert_eq!(hist.n_bins(), 3); + assert_eq!(hist.n_samples(), 2); } #[test] @@ -1624,4 +2106,14 @@ mod tests { "Histogram reconstructed from paired arrays is not correct" ); } + #[test] + fn test_parse_bytes_measurement() { + let s = r#"{"timeseries_key": 101, "timestamp": "2023-11-21 18:25:21.963714255", "datum": "\u0001\u0002\u0003"}"#; + let (_, meas) = parse_timeseries_scalar_gauge_measurement::(&s); + println!("{meas:?}"); + let Datum::Bytes(b) = meas.datum() else { + unreachable!(); + }; + assert_eq!(b.to_vec(), vec![1, 2, 3]); + } } diff --git a/oximeter/oximeter/Cargo.toml b/oximeter/oximeter/Cargo.toml index 8a69494d5a..0cb2d8cace 100644 --- a/oximeter/oximeter/Cargo.toml +++ b/oximeter/oximeter/Cargo.toml @@ -21,4 +21,5 @@ omicron-workspace-hack.workspace = true [dev-dependencies] approx.workspace = true rstest.workspace = true +serde_json.workspace = true trybuild.workspace = true diff --git a/oximeter/oximeter/src/histogram.rs b/oximeter/oximeter/src/histogram.rs index c399384ffa..aaf9297ca4 100644 --- a/oximeter/oximeter/src/histogram.rs +++ b/oximeter/oximeter/src/histogram.rs @@ -1353,13 +1353,10 @@ mod tests { } #[test] - fn test_foo() { - let bins: Vec = 10u16.bins(1, 3, 30.try_into().unwrap()).unwrap(); - println!("{bins:?}"); - dbg!(bins.len()); - let hist = Histogram::new(&bins).unwrap(); - for bin in hist.iter() { - println!("{}", bin.range); - } + fn test_empty_bins_not_supported() { + assert!(matches!( + Histogram::::new(&[]).unwrap_err(), + HistogramError::EmptyBins + )); } } diff --git a/oximeter/oximeter/src/test_util.rs b/oximeter/oximeter/src/test_util.rs index f3750d6d83..a9778d03bc 100644 --- a/oximeter/oximeter/src/test_util.rs +++ b/oximeter/oximeter/src/test_util.rs @@ -48,19 +48,27 @@ pub struct TestHistogram { pub datum: Histogram, } +const ID: Uuid = uuid::uuid!("e00ced4d-39d1-446a-ae85-a67f05c9750b"); + pub fn make_sample() -> Sample { let target = TestTarget::default(); - let metric = TestMetric { id: Uuid::new_v4(), good: true, datum: 1 }; + let metric = TestMetric { id: ID, good: true, datum: 1 }; Sample::new(&target, &metric).unwrap() } +pub fn make_missing_sample() -> Sample { + let target = TestTarget::default(); + let metric = TestMetric { id: ID, good: true, datum: 1 }; + Sample::new_missing(&target, &metric).unwrap() +} + pub fn make_hist_sample() -> Sample { let target = TestTarget::default(); let mut hist = histogram::Histogram::new(&[0.0, 5.0, 10.0]).unwrap(); hist.sample(1.0).unwrap(); hist.sample(2.0).unwrap(); hist.sample(6.0).unwrap(); - let metric = TestHistogram { id: Uuid::new_v4(), good: true, datum: hist }; + let metric = TestHistogram { id: ID, good: true, datum: hist }; Sample::new(&target, &metric).unwrap() } diff --git a/oximeter/oximeter/src/traits.rs b/oximeter/oximeter/src/traits.rs index 096abb8023..0934d231e3 100644 --- a/oximeter/oximeter/src/traits.rs +++ b/oximeter/oximeter/src/traits.rs @@ -30,8 +30,15 @@ use std::ops::AddAssign; /// definition can be thought of as a schema, and an instance of that struct as identifying an /// individual target. /// -/// Target fields may have one of a set of supported types: `bool`, `i64`, `String`, `IpAddr`, or -/// `Uuid`. Any number of fields greater than zero is supported. +/// Target fields may have one of a set of supported types: +/// +/// - `bool` +/// - any fixed-width integer, e.g., `u8` or `i64` +/// - `String` +/// - `IpAddr` +/// - `Uuid` +/// +/// Any number of fields greater than zero is supported. /// /// Examples /// -------- @@ -105,9 +112,28 @@ pub trait Target { /// One field of the struct is special, describing the actual measured data that the metric /// represents. This should be a field named `datum`, or another field (with any name you choose) /// annotated with the `#[datum]` attribute. This field represents the underlying data for the -/// metric, and must be one of the supported types, implementing the [`Datum`] trait. This can -/// be any of: `i64`, `f64`, `bool`, `String`, or `Bytes` for gauges, and `Cumulative` or -/// `Histogram` for cumulative metrics, where `T` is `i64` or `f64`. +/// metric, and must be one of the supported types, implementing the [`Datum`] trait. +/// +/// For gauge types, this can be any of: +/// +/// - `bool` +/// - a fixed-width integer, e.g. `u8` or `i64` +/// - `f32` or `f64` +/// - `String` +/// - `Bytes` +/// +/// Cumulative types can be any of `Cumulative`, where `T` is +/// +/// - `i64` +/// - `u64` +/// - `f32` +/// - `f64` +/// +/// Histogram types can be any `Histogram`, wher `T` is: +/// +/// - a fixed-width integer, e.g. `u8` or `i64` +/// - `f32` +/// - `f64` /// /// The value of the metric's data is _measured_ by using the `measure()` method, which returns a /// [`Measurement`]. This describes a timestamped data point for the metric. diff --git a/oximeter/oximeter/src/types.rs b/oximeter/oximeter/src/types.rs index 325974781e..23dbe2be6b 100644 --- a/oximeter/oximeter/src/types.rs +++ b/oximeter/oximeter/src/types.rs @@ -369,6 +369,7 @@ pub enum Datum { HistogramU64(histogram::Histogram), HistogramF32(histogram::Histogram), HistogramF64(histogram::Histogram), + Missing(MissingDatum), } impl Datum { @@ -402,6 +403,7 @@ impl Datum { Datum::HistogramU64(_) => DatumType::HistogramU64, Datum::HistogramF32(_) => DatumType::HistogramF32, Datum::HistogramF64(_) => DatumType::HistogramF64, + Datum::Missing(ref inner) => inner.datum_type(), } } @@ -440,6 +442,7 @@ impl Datum { Datum::HistogramU64(ref inner) => Some(inner.start_time()), Datum::HistogramF32(ref inner) => Some(inner.start_time()), Datum::HistogramF64(ref inner) => Some(inner.start_time()), + Datum::Missing(ref inner) => inner.start_time(), } } } @@ -495,6 +498,60 @@ impl From<&str> for Datum { } } +#[derive(Clone, Copy, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +pub struct MissingDatum { + datum_type: DatumType, + start_time: Option>, +} + +impl MissingDatum { + pub fn datum_type(&self) -> DatumType { + self.datum_type + } + + pub fn start_time(&self) -> Option> { + self.start_time + } + + pub fn new( + datum_type: DatumType, + start_time: Option>, + ) -> Result { + // See https://github.com/oxidecomputer/omicron/issues/4551. + if datum_type == DatumType::Bytes { + return Err(MetricsError::DatumError(String::from( + "Missing samples from byte array types are not supported", + ))); + } + if datum_type.is_cumulative() && start_time.is_none() { + return Err(MetricsError::MissingDatumRequiresStartTime { + datum_type, + }); + } + if !datum_type.is_cumulative() && start_time.is_some() { + return Err(MetricsError::MissingDatumCannotHaveStartTime { + datum_type, + }); + } + Ok(Self { datum_type, start_time }) + } +} + +impl From for Datum { + fn from(d: MissingDatum) -> Datum { + Datum::Missing(d) + } +} + +impl From<&M> for MissingDatum { + fn from(metric: &M) -> Self { + MissingDatum { + datum_type: metric.datum_type(), + start_time: metric.start_time(), + } + } +} + /// A `Measurement` is a timestamped datum from a single metric #[derive(Clone, Debug, PartialEq, JsonSchema, Serialize, Deserialize)] pub struct Measurement { @@ -516,6 +573,11 @@ impl Measurement { Self { timestamp, datum: datum.into() } } + /// Return true if this measurement represents a missing datum. + pub fn is_missing(&self) -> bool { + matches!(self.datum, Datum::Missing(_)) + } + /// Return the datum for this measurement pub fn datum(&self) -> &Datum { &self.datum @@ -561,6 +623,12 @@ pub enum MetricsError { /// A field name is duplicated between the target and metric. #[error("Field '{name}' is duplicated between the target and metric")] DuplicateFieldName { name: String }, + + #[error("Missing datum of type {datum_type} requires a start time")] + MissingDatumRequiresStartTime { datum_type: DatumType }, + + #[error("Missing datum of type {datum_type} cannot have a start time")] + MissingDatumCannotHaveStartTime { datum_type: DatumType }, } impl From for omicron_common::api::external::Error { @@ -734,6 +802,29 @@ impl Sample { }) } + /// Construct a new missing sample, recorded at the time of the supplied + /// timestamp. + pub fn new_missing_with_timestamp( + timestamp: DateTime, + target: &T, + metric: &M, + ) -> Result + where + T: traits::Target, + M: traits::Metric, + { + let target_fields = FieldSet::from_target(target); + let metric_fields = FieldSet::from_metric(metric); + Self::verify_field_names(&target_fields, &metric_fields)?; + let datum = Datum::Missing(MissingDatum::from(metric)); + Ok(Self { + timeseries_name: crate::timeseries_name(target, metric), + target: target_fields, + metric: metric_fields, + measurement: Measurement { timestamp, datum }, + }) + } + /// Construct a new sample, created at the time the function is called. /// /// This materializes the data from the target and metric, and stores that information along @@ -746,6 +837,18 @@ impl Sample { Self::new_with_timestamp(Utc::now(), target, metric) } + /// Construct a new sample with a missing measurement. + pub fn new_missing( + target: &T, + metric: &M, + ) -> Result + where + T: traits::Target, + M: traits::Metric, + { + Self::new_missing_with_timestamp(Utc::now(), target, metric) + } + /// Return the fields for this sample. /// /// This returns the target fields and metric fields, chained, although there is no distinction @@ -951,7 +1054,7 @@ mod tests { fn test_measurement() { let measurement = Measurement::new(chrono::Utc::now(), 0i64); assert_eq!(measurement.datum_type(), DatumType::I64); - assert_eq!(measurement.start_time(), None); + assert!(measurement.start_time().is_none()); let datum = Cumulative::new(0i64); let measurement = Measurement::new(chrono::Utc::now(), datum); From 5c90213e1b07b64cb02b9f5d2858c09f2832371a Mon Sep 17 00:00:00 2001 From: Rain Date: Mon, 4 Dec 2023 16:09:20 -0800 Subject: [PATCH 08/16] [nexus] remove views::SledProvisionState::Unknown (#4608) This doesn't quite work as expected: * As an input type, if an unknown `SledProvisionState` is specified, we immediately produce an error as soon as we enter the HTTP entrypoint. There's no functional difference between that and producing an error at deserialization time. * As an output type, progenitor doesn't support `#[serde(other)]` so the unknown type doesn't work. --- Cargo.lock | 1 - nexus/db-model/src/sled_provision_state.rs | 13 ++++--------- nexus/src/external_api/http_entrypoints.rs | 5 +---- nexus/types/Cargo.toml | 1 - nexus/types/src/external_api/views.rs | 7 ------- openapi/nexus.json | 7 ------- 6 files changed, 5 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28d3015025..9671cb34bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4179,7 +4179,6 @@ dependencies = [ "schemars", "serde", "serde_json", - "serde_with", "steno", "strum", "uuid", diff --git a/nexus/db-model/src/sled_provision_state.rs b/nexus/db-model/src/sled_provision_state.rs index 6cf81b9c70..b2b1ee39dc 100644 --- a/nexus/db-model/src/sled_provision_state.rs +++ b/nexus/db-model/src/sled_provision_state.rs @@ -34,19 +34,14 @@ impl From for views::SledProvisionState { } } -impl TryFrom for SledProvisionState { - type Error = UnknownSledProvisionState; - - fn try_from(state: views::SledProvisionState) -> Result { +impl From for SledProvisionState { + fn from(state: views::SledProvisionState) -> Self { match state { views::SledProvisionState::Provisionable => { - Ok(SledProvisionState::Provisionable) + SledProvisionState::Provisionable } views::SledProvisionState::NonProvisionable => { - Ok(SledProvisionState::NonProvisionable) - } - views::SledProvisionState::Unknown => { - Err(UnknownSledProvisionState) + SledProvisionState::NonProvisionable } } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index f1302f4a73..ef8d73afab 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -4504,10 +4504,7 @@ async fn sled_set_provision_state( let opctx = crate::context::op_context_for_external_api(&rqctx).await?; // Convert the external `SledProvisionState` into our internal data model. - let new_state = - db::model::SledProvisionState::try_from(provision_state).map_err( - |error| HttpError::for_bad_request(None, format!("{error}")), - )?; + let new_state = db::model::SledProvisionState::from(provision_state); let sled_lookup = nexus.sled_lookup(&opctx, &path.sled_id)?; diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index 8cbbd8626c..9cb94a8484 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -14,7 +14,6 @@ parse-display.workspace = true schemars = { workspace = true, features = ["chrono", "uuid1"] } serde.workspace = true serde_json.workspace = true -serde_with.workspace = true steno.workspace = true strum.workspace = true uuid.workspace = true diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 6d02623f34..4006b18bcc 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -17,7 +17,6 @@ use omicron_common::api::external::{ }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use serde_with::rust::deserialize_ignore_any; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::net::IpAddr; @@ -311,12 +310,6 @@ pub enum SledProvisionState { /// resources will continue to be on this sled unless manually migrated /// off. NonProvisionable, - - /// This is a state that isn't known yet. - /// - /// This is defined to avoid API breakage. - #[serde(other, deserialize_with = "deserialize_ignore_any")] - Unknown, } /// An operator's view of an instance running on a given sled diff --git a/openapi/nexus.json b/openapi/nexus.json index a6dffc6265..1c7e25d004 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -13259,13 +13259,6 @@ "enum": [ "non_provisionable" ] - }, - { - "description": "This is a state that isn't known yet.\n\nThis is defined to avoid API breakage.", - "type": "string", - "enum": [ - "unknown" - ] } ] }, From 301edd733bbbfb3117ad4ae30c483fb6e97ea0c7 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Tue, 5 Dec 2023 05:14:35 +0000 Subject: [PATCH 09/16] Update taiki-e/install-action digest to d211c4b (#4612) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [taiki-e/install-action](https://togithub.com/taiki-e/install-action) | action | digest | [`21526ba` -> `d211c4b`](https://togithub.com/taiki-e/install-action/compare/21526ba...d211c4b) | --- ### Configuration 📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone America/Los_Angeles. 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://togithub.com/renovatebot/renovate). Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com> --- .github/workflows/hakari.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index b5a7504066..70b57a450a 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -24,7 +24,7 @@ jobs: with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@21526ba3bb38834e625c185ae4f2f942f1fb8f27 # v2 + uses: taiki-e/install-action@d211c4be5a95cbcd52a0870dda7d63a107a58368 # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date From 0a781f9957f5da1ef4b2ca7dce1d7bf13be5b739 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:15:21 -0800 Subject: [PATCH 10/16] Update Rust crate derive-where to 1.2.6 (#4613) Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9671cb34bb..8f855f5219 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1585,9 +1585,9 @@ dependencies = [ [[package]] name = "derive-where" -version = "1.2.5" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146398d62142a0f35248a608f17edf0dde57338354966d6e41d0eb2d16980ccb" +checksum = "48d9b1fc2a6d7e19c89e706a3769e31ee862ac7a4c810c7c0ff3910e1a42a4ce" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 533e710dc2..931d885e7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -181,7 +181,7 @@ ddm-admin-client = { path = "clients/ddm-admin-client" } db-macros = { path = "nexus/db-macros" } debug-ignore = "1.0.5" derive_more = "0.99.17" -derive-where = "1.2.5" +derive-where = "1.2.6" diesel = { version = "2.1.4", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } diesel-dtrace = { git = "https://github.com/oxidecomputer/diesel-dtrace", branch = "main" } dns-server = { path = "dns-server" } From 7dca6fc9edf52fbf6846deec9346217c495c235d Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Tue, 5 Dec 2023 06:59:04 -0800 Subject: [PATCH 11/16] [wicketd] Add refresh-config subcommand instead of using curl (#4606) This allows us to add some time for refreshing a wicketd that hasn't fully started (and therefore isn't reachable on its dropshot server yet). Fixes https://github.com/oxidecomputer/omicron/issues/4604. Testing this is a little awkward because config refresh is only support on illumos via SMF, so I tested this by hand on `madrid`: 0. Disable wicketd (`svcadm disable wicketd`) 1. Install the new wicketd binary and `manifest.xml` in the switch zone 2. Import the new manifest.xml (`svccfg import /var/svc/manifest/site/wicketd/manifest.xml`) 3. (I expected to need to restore the config properties that sled-agent had set, but they persisted) 4. Enable wicketd and immediately try to refresh it (`svcadm enable wicketd && svcadm refresh wicketd`) In the wicketd logs, we see that the first refresh attempt failed because it fired before the dropshot server was up: ``` [ Dec 4 18:34:22 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/wicketd/bin/wicketd run /var/svc/manifest/site/wicketd/config.toml --address [::1]:12226 --artifact-address [fdb0:a840:2504:355::2]:12227 --mgs-address [::1]:12225 --nex us-proxy-address [::]:12229 --baseboard-file /opt/oxide/baseboard.json --read-smf-config &"). ] [ Dec 4 18:34:22 Method "start" exited with status 0. ] [ Dec 4 18:34:22 Rereading configuration. ] note: configured to log to "/dev/stdout" [ Dec 4 18:34:22 Executing refresh method ("/opt/oxide/wicketd/bin/wicketd refresh-config /var/svc/manifest/site/wicketd/config.toml --address [::1]:12226"). ] note: configured to log to "/dev/stdout" 18:34:22.330Z WARN wicketd: failed to refresh wicketd config (attempt 1 of 3); will retry after 5s err = Communication Error: error sending request for url (http://[::1]:12226/reload-config): error trying to connect: tcp connect error: Connection refused (os error 146)\nCaused by:\n -> error sending request for url (http://[::1]:12226/reload-config) : error trying to connect: tcp connect error: Connection refused (os error 146)\n -> error trying to connect: tcp connect error: Connection refused (os error 146)\n -> tcp connect error: Connection refused (os error 146)\n -> Connection refused (os error 146) 18:34:22.396Z INFO wicketd (dropshot (wicketd)): listening file = /home/john/.cargo/git/checkouts/dropshot-a4a923d29dccc492/ff87a01/dropshot/src/server.rs:195 local_addr = [::1]:12226 ``` 10 seconds later, we see the successful connection, POST, and exit of the SMF `refresh`: ``` 18:34:32.332Z INFO wicketd (dropshot (wicketd)): accepted connection file = /home/john/.cargo/git/checkouts/dropshot-a4a923d29dccc492/ff87a01/dropshot/src/server.rs:769 local_addr = [::1]:12226 remote_addr = [::1]:32976 18:34:32.388Z INFO wicketd (dropshot (wicketd)): request completed file = /home/john/.cargo/git/checkouts/dropshot-a4a923d29dccc492/ff87a01/dropshot/src/server.rs:853 latency_us = 30475 local_addr = [::1]:12226 method = POST remote_addr = [::1]:32976 req_id = e0c2034a-0a99-45c1-a651-57249ca258f0 response_code = 204 uri = /reload-config [ Dec 4 18:34:32 Method "refresh" exited with status 0. ] ``` --- smf/wicketd/manifest.xml | 2 +- wicketd/Cargo.toml | 2 +- wicketd/src/bin/wicketd.rs | 41 ++++++++++++++++++++--- wicketd/src/lib.rs | 68 +++++++++++++++++++++++++++++++++++--- 4 files changed, 102 insertions(+), 11 deletions(-) diff --git a/smf/wicketd/manifest.xml b/smf/wicketd/manifest.xml index 778a7abf2d..b45ff1544b 100644 --- a/smf/wicketd/manifest.xml +++ b/smf/wicketd/manifest.xml @@ -32,7 +32,7 @@ it expected https). --> diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index 1360c28b19..97550342d0 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -58,6 +58,7 @@ sled-hardware.workspace = true tufaceous-lib.workspace = true update-engine.workspace = true wicket-common.workspace = true +wicketd-client.workspace = true omicron-workspace-hack.workspace = true [[bin]] @@ -83,4 +84,3 @@ tar.workspace = true tokio = { workspace = true, features = ["test-util"] } tufaceous.workspace = true wicket.workspace = true -wicketd-client.workspace = true diff --git a/wicketd/src/bin/wicketd.rs b/wicketd/src/bin/wicketd.rs index 887ac496e0..24fa802c79 100644 --- a/wicketd/src/bin/wicketd.rs +++ b/wicketd/src/bin/wicketd.rs @@ -5,6 +5,7 @@ //! Executable for wicketd: technician port based management service use anyhow::{anyhow, Context}; +use camino::Utf8PathBuf; use clap::Parser; use omicron_common::{ address::Ipv6Subnet, @@ -24,9 +25,9 @@ enum Args { /// Start a wicketd server Run { #[clap(name = "CONFIG_FILE_PATH", action)] - config_file_path: PathBuf, + config_file_path: Utf8PathBuf, - /// The address for the technician port + /// The address on which the main wicketd dropshot server should listen #[clap(short, long, action)] address: SocketAddrV6, @@ -57,6 +58,19 @@ enum Args { #[clap(long, action, conflicts_with("read_smf_config"))] rack_subnet: Option, }, + + /// Instruct a running wicketd server to refresh its config + /// + /// Mechanically, this hits a specific endpoint served by wicketd's dropshot + /// server + RefreshConfig { + #[clap(name = "CONFIG_FILE_PATH", action)] + config_file_path: Utf8PathBuf, + + /// The address of the server to refresh + #[clap(short, long, action)] + address: SocketAddrV6, + }, } #[tokio::main] @@ -104,9 +118,7 @@ async fn do_run() -> Result<(), CmdError> { }; let config = Config::from_file(&config_file_path) - .with_context(|| { - format!("failed to parse {}", config_file_path.display()) - }) + .with_context(|| format!("failed to parse {config_file_path}")) .map_err(CmdError::Failure)?; let rack_subnet = match rack_subnet { @@ -140,5 +152,24 @@ async fn do_run() -> Result<(), CmdError> { .await .map_err(|err| CmdError::Failure(anyhow!(err))) } + Args::RefreshConfig { config_file_path, address } => { + let config = Config::from_file(&config_file_path) + .with_context(|| format!("failed to parse {config_file_path}")) + .map_err(CmdError::Failure)?; + + let log = config + .log + .to_logger("wicketd") + .context("failed to initialize logger") + .map_err(CmdError::Failure)?; + + // When run via `svcadm refresh ...`, we need to respect the special + // [SMF exit codes](https://illumos.org/man/7/smf_method). Returning + // an error from main exits with code 1 (from libc::EXIT_FAILURE), + // which does not collide with any special SMF codes. + Server::refresh_config(log, address) + .await + .map_err(CmdError::Failure) + } } } diff --git a/wicketd/src/lib.rs b/wicketd/src/lib.rs index ada1902654..32188d77de 100644 --- a/wicketd/src/lib.rs +++ b/wicketd/src/lib.rs @@ -16,11 +16,12 @@ mod preflight_check; mod rss_config; mod update_tracker; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use artifacts::{WicketdArtifactServer, WicketdArtifactStore}; use bootstrap_addrs::BootstrapPeers; pub use config::Config; pub(crate) use context::ServerContext; +use display_error_chain::DisplayErrorChain; use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer}; pub use installinator_progress::{IprUpdateTracker, RunningUpdateState}; use internal_dns::resolver::Resolver; @@ -34,6 +35,7 @@ use preflight_check::PreflightCheckerHandler; use sled_hardware::Baseboard; use slog::{debug, error, o, Drain}; use std::sync::{Mutex, OnceLock}; +use std::time::Duration; use std::{ net::{SocketAddr, SocketAddrV6}, sync::Arc, @@ -70,7 +72,6 @@ pub struct SmfConfigValues { impl SmfConfigValues { #[cfg(target_os = "illumos")] pub fn read_current() -> Result { - use anyhow::Context; use illumos_utils::scf::ScfHandle; const CONFIG_PG: &str = "config"; @@ -259,11 +260,70 @@ impl Server { res = self.artifact_server => { match res { Ok(()) => Err("artifact server exited unexpectedly".to_owned()), - // The artifact server returns an anyhow::Error, which has a `Debug` impl that - // prints out the chain of errors. + // The artifact server returns an anyhow::Error, which has a + // `Debug` impl that prints out the chain of errors. Err(err) => Err(format!("running artifact server: {err:?}")), } } } } + + /// Instruct a running server at the specified address to reload its config + /// parameters + pub async fn refresh_config( + log: slog::Logger, + address: SocketAddrV6, + ) -> Result<()> { + // It's possible we're being told to refresh a server's config before + // it's ready to receive such a request, so we'll give it a healthy + // amount of time before we give up: we'll set a client timeout and also + // retry a few times. See + // https://github.com/oxidecomputer/omicron/issues/4604. + const CLIENT_TIMEOUT: Duration = Duration::from_secs(5); + const SLEEP_BETWEEN_RETRIES: Duration = Duration::from_secs(10); + const NUM_RETRIES: usize = 3; + + let client = reqwest::Client::builder() + .connect_timeout(CLIENT_TIMEOUT) + .timeout(CLIENT_TIMEOUT) + .build() + .context("failed to construct reqwest Client")?; + + let client = wicketd_client::Client::new_with_client( + &format!("http://{address}"), + client, + log, + ); + let log = client.inner(); + + let mut attempt = 0; + loop { + attempt += 1; + + // If we succeed, we're done. + let Err(err) = client.post_reload_config().await else { + return Ok(()); + }; + + // If we failed, either warn+sleep and try again, or fail. + if attempt < NUM_RETRIES { + slog::warn!( + log, + "failed to refresh wicketd config \ + (attempt {attempt} of {NUM_RETRIES}); \ + will retry after {CLIENT_TIMEOUT:?}"; + "err" => %DisplayErrorChain::new(&err), + ); + tokio::time::sleep(SLEEP_BETWEEN_RETRIES).await; + } else { + slog::error!( + log, + "failed to refresh wicketd config \ + (tried {NUM_RETRIES} times)"; + "err" => %DisplayErrorChain::new(&err), + ); + return Err(err).context("failed to contact wicketd"); + } + } + } } From 4b426d260a06e436f2b398e99a602568d8026e6e Mon Sep 17 00:00:00 2001 From: Rain Date: Tue, 5 Dec 2023 12:55:52 -0800 Subject: [PATCH 12/16] [ci] update nextest to 0.9.64 (#4609) There have been a bunch of releases and I don't want us to fall too far behind in CI. (Maybe this should be handled by Renovate :) ) --- .config/nextest.toml | 2 +- .github/buildomat/build-and-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index ef296d7ef8..4f927d2396 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -3,7 +3,7 @@ # # The required version should be bumped up if we need new features, performance # improvements or bugfixes that are present in newer versions of nextest. -nextest-version = { required = "0.9.59", recommended = "0.9.59" } +nextest-version = { required = "0.9.59", recommended = "0.9.64" } experimental = ["setup-scripts"] diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index 6fda8bb8d7..34f81bab68 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -7,7 +7,7 @@ set -o xtrace # NOTE: This version should be in sync with the recommended version in # .config/nextest.toml. (Maybe build an automated way to pull the recommended # version in the future.) -NEXTEST_VERSION='0.9.59' +NEXTEST_VERSION='0.9.64' cargo --version rustc --version From d525deef3f12a08fc836bb10e8ad2ce7661fa66b Mon Sep 17 00:00:00 2001 From: iliana etaoin Date: Tue, 5 Dec 2023 13:52:06 -0800 Subject: [PATCH 13/16] Set version to 5.0.0 (#4566) --- .github/buildomat/jobs/package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 0605ab6883..350ab37233 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -37,7 +37,7 @@ rustc --version # trampoline global zone images. # COMMIT=$(git rev-parse HEAD) -VERSION="1.0.4-0.ci+git${COMMIT:0:11}" +VERSION="5.0.0-0.ci+git${COMMIT:0:11}" echo "$VERSION" >/work/version.txt ptime -m ./tools/install_builder_prerequisites.sh -yp From b3d641a2dd3bb3f3dd68a413ddd670e3dbcc0b5c Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 5 Dec 2023 20:48:26 -0800 Subject: [PATCH 14/16] [nexus] Make most transactions automatically retry (#4487) Integrates automatic transaction retry into Nexus for most transactions. Additionally, this PR provides a "RetryHelper" object to help standardize how transaction retry is performed. Currently, after a short randomized wait (up to an upper bound), we retry unconditionally, emitting each attempt to Oximeter for further analysis. - [x] Depends on https://github.com/oxidecomputer/async-bb8-diesel/pull/58 - [x] As noted in https://github.com/oxidecomputer/async-bb8-diesel/pull/58, this will require customizing CRDB session variables to work correctly. (Edit: this is done on each transaction) Part of https://github.com/oxidecomputer/customer-support/issues/46 Part of https://github.com/oxidecomputer/omicron/issues/3814 --- Cargo.lock | 3 +- Cargo.toml | 2 +- nexus/db-model/src/sled.rs | 2 +- nexus/db-queries/Cargo.toml | 1 + nexus/db-queries/src/db/collection_attach.rs | 27 +- .../src/db/collection_detach_many.rs | 26 +- .../src/db/datastore/address_lot.rs | 103 +- nexus/db-queries/src/db/datastore/bgp.rs | 424 +++--- .../src/db/datastore/db_metadata.rs | 47 +- .../src/db/datastore/device_auth.rs | 53 +- nexus/db-queries/src/db/datastore/dns.rs | 80 +- .../src/db/datastore/external_ip.rs | 24 +- .../src/db/datastore/identity_provider.rs | 68 +- nexus/db-queries/src/db/datastore/mod.rs | 26 + .../src/db/datastore/network_interface.rs | 129 +- nexus/db-queries/src/db/datastore/project.rs | 172 ++- nexus/db-queries/src/db/datastore/rack.rs | 514 ++++--- nexus/db-queries/src/db/datastore/region.rs | 165 +-- nexus/db-queries/src/db/datastore/service.rs | 40 +- nexus/db-queries/src/db/datastore/silo.rs | 151 +- .../db-queries/src/db/datastore/silo_group.rs | 54 +- nexus/db-queries/src/db/datastore/sled.rs | 245 ++-- nexus/db-queries/src/db/datastore/snapshot.rs | 180 ++- .../src/db/datastore/switch_interface.rs | 166 ++- .../src/db/datastore/switch_port.rs | 1294 +++++++++-------- nexus/db-queries/src/db/datastore/update.rs | 56 +- .../virtual_provisioning_collection.rs | 32 +- nexus/db-queries/src/db/datastore/volume.rs | 685 +++++---- nexus/db-queries/src/db/datastore/vpc.rs | 75 +- nexus/db-queries/src/db/error.rs | 70 +- nexus/db-queries/src/db/mod.rs | 4 +- .../src/db/queries/network_interface.rs | 15 +- nexus/db-queries/src/lib.rs | 1 + nexus/db-queries/src/transaction_retry.rs | 341 +++++ nexus/src/app/background/dns_config.rs | 11 +- nexus/src/app/background/init.rs | 8 +- nexus/src/app/sagas/disk_create.rs | 32 +- nexus/src/app/sagas/instance_create.rs | 83 +- nexus/src/app/sagas/project_create.rs | 15 +- nexus/src/app/sagas/test_helpers.rs | 25 +- 40 files changed, 3003 insertions(+), 2446 deletions(-) create mode 100644 nexus/db-queries/src/transaction_retry.rs diff --git a/Cargo.lock b/Cargo.lock index 8f855f5219..13b3d6c74e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -250,7 +250,7 @@ checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" [[package]] name = "async-bb8-diesel" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=1446f7e0c1f05f33a0581abd51fa873c7652ab61#1446f7e0c1f05f33a0581abd51fa873c7652ab61" +source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=ed7ab5ef0513ba303d33efd41d3e9e381169d59b#ed7ab5ef0513ba303d33efd41d3e9e381169d59b" dependencies = [ "async-trait", "bb8", @@ -4047,6 +4047,7 @@ dependencies = [ "pem 1.1.1", "petgraph", "pq-sys", + "rand 0.8.5", "rcgen", "ref-cast", "regex", diff --git a/Cargo.toml b/Cargo.toml index 931d885e7a..c88502bb1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,7 +144,7 @@ api_identity = { path = "api_identity" } approx = "0.5.1" assert_matches = "1.5.0" assert_cmd = "2.0.12" -async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "1446f7e0c1f05f33a0581abd51fa873c7652ab61" } +async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "ed7ab5ef0513ba303d33efd41d3e9e381169d59b" } async-trait = "0.1.74" atomicwrites = "0.4.2" authz-macros = { path = "nexus/authz-macros" } diff --git a/nexus/db-model/src/sled.rs b/nexus/db-model/src/sled.rs index 0f6d1b911e..85a6b3139c 100644 --- a/nexus/db-model/src/sled.rs +++ b/nexus/db-model/src/sled.rs @@ -232,7 +232,7 @@ impl SledUpdate { } /// A set of constraints that can be placed on operations that select a sled. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct SledReservationConstraints { must_select_from: Vec, } diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index 94e3a56abf..9d8afd1fea 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -32,6 +32,7 @@ oso.workspace = true paste.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. pq-sys = "*" +rand.workspace = true ref-cast.workspace = true samael.workspace = true serde.workspace = true diff --git a/nexus/db-queries/src/db/collection_attach.rs b/nexus/db-queries/src/db/collection_attach.rs index ea4d9d5beb..fccc1aa324 100644 --- a/nexus/db-queries/src/db/collection_attach.rs +++ b/nexus/db-queries/src/db/collection_attach.rs @@ -563,12 +563,9 @@ where #[cfg(test)] mod test { use super::*; - use crate::db::{ - self, error::TransactionError, identity::Resource as IdentityResource, - }; + use crate::db::{self, identity::Resource as IdentityResource}; use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - ConnectionManager, + AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, }; use chrono::Utc; use db_macros::Resource; @@ -999,22 +996,12 @@ mod test { .set(resource::dsl::collection_id.eq(collection_id)), ); - type TxnError = - TransactionError>; - let result = conn - .transaction_async(|conn| async move { - attach_query.attach_and_get_result_async(&conn).await.map_err( - |e| match e { - AttachError::DatabaseError(e) => TxnError::from(e), - e => TxnError::CustomError(e), - }, - ) - }) - .await; - // "attach_and_get_result" should return the "attached" resource. - let (returned_collection, returned_resource) = - result.expect("Attach should have worked"); + let (returned_collection, returned_resource) = attach_query + .attach_and_get_result_async(&conn) + .await + .expect("Attach should have worked"); + assert_eq!( returned_resource.collection_id.expect("Expected a collection ID"), collection_id diff --git a/nexus/db-queries/src/db/collection_detach_many.rs b/nexus/db-queries/src/db/collection_detach_many.rs index 8df6d4aed4..986cfb70b7 100644 --- a/nexus/db-queries/src/db/collection_detach_many.rs +++ b/nexus/db-queries/src/db/collection_detach_many.rs @@ -479,12 +479,9 @@ where mod test { use super::*; use crate::db::collection_attach::DatastoreAttachTarget; - use crate::db::{ - self, error::TransactionError, identity::Resource as IdentityResource, - }; + use crate::db::{self, identity::Resource as IdentityResource}; use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - ConnectionManager, + AsyncRunQueryDsl, AsyncSimpleConnection, ConnectionManager, }; use chrono::Utc; use db_macros::Resource; @@ -919,21 +916,12 @@ mod test { .set(resource::dsl::collection_id.eq(Option::::None)), ); - type TxnError = - TransactionError>; - let result = conn - .transaction_async(|conn| async move { - detach_query.detach_and_get_result_async(&conn).await.map_err( - |e| match e { - DetachManyError::DatabaseError(e) => TxnError::from(e), - e => TxnError::CustomError(e), - }, - ) - }) - .await; - // "detach_and_get_result" should return the "detached" resource. - let returned_collection = result.expect("Detach should have worked"); + let returned_collection = detach_query + .detach_and_get_result_async(&conn) + .await + .expect("Detach should have worked"); + // The returned values should be the latest value in the DB. assert_eq!( returned_collection, diff --git a/nexus/db-queries/src/db/datastore/address_lot.rs b/nexus/db-queries/src/db/datastore/address_lot.rs index 97dfb59eba..5c2ffbf1d0 100644 --- a/nexus/db-queries/src/db/datastore/address_lot.rs +++ b/nexus/db-queries/src/db/datastore/address_lot.rs @@ -13,9 +13,9 @@ use crate::db::error::TransactionError; use crate::db::model::Name; use crate::db::model::{AddressLot, AddressLotBlock, AddressLotReservedBlock}; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl, Connection}; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::{AsyncRunQueryDsl, Connection}; use chrono::Utc; -use diesel::result::Error as DieselError; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use diesel_dtrace::DTraceConnection; use ipnetwork::IpNetwork; @@ -45,11 +45,12 @@ impl DataStore { use db::schema::address_lot::dsl as lot_dsl; use db::schema::address_lot_block::dsl as block_dsl; - self.pool_connection_authorized(opctx) - .await? - // TODO https://github.com/oxidecomputer/omicron/issues/2811 - // Audit external networking database transaction usage - .transaction_async(|conn| async move { + let conn = self.pool_connection_authorized(opctx).await?; + + // TODO https://github.com/oxidecomputer/omicron/issues/2811 + // Audit external networking database transaction usage + self.transaction_retry_wrapper("address_lot_create") + .transaction(&conn, |conn| async move { let lot = AddressLot::new(¶ms.identity, params.kind.into()); let db_lot: AddressLot = @@ -81,15 +82,14 @@ impl DataStore { Ok(AddressLotCreateResult { lot: db_lot, blocks: db_blocks }) }) .await - .map_err(|e| match e { - DieselError::DatabaseError(_, _) => public_error_from_diesel( + .map_err(|e| { + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::AddressLot, ¶ms.identity.name.as_str(), ), - ), - _ => public_error_from_diesel(e, ErrorHandler::Server), + ) }) } @@ -113,47 +113,54 @@ impl DataStore { LotInUse, } - type TxnError = TransactionError; + let err = OptionalError::new(); // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - let rsvd: Vec = - rsvd_block_dsl::address_lot_rsvd_block - .filter(rsvd_block_dsl::address_lot_id.eq(id)) - .select(AddressLotReservedBlock::as_select()) - .limit(1) - .load_async(&conn) - .await?; - - if !rsvd.is_empty() { - Err(TxnError::CustomError(AddressLotDeleteError::LotInUse))?; - } + self.transaction_retry_wrapper("address_lot_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let rsvd: Vec = + rsvd_block_dsl::address_lot_rsvd_block + .filter(rsvd_block_dsl::address_lot_id.eq(id)) + .select(AddressLotReservedBlock::as_select()) + .limit(1) + .load_async(&conn) + .await?; + + if !rsvd.is_empty() { + return Err(err.bail(AddressLotDeleteError::LotInUse)); + } + + let now = Utc::now(); + diesel::update(lot_dsl::address_lot) + .filter(lot_dsl::time_deleted.is_null()) + .filter(lot_dsl::id.eq(id)) + .set(lot_dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await?; - let now = Utc::now(); - diesel::update(lot_dsl::address_lot) - .filter(lot_dsl::time_deleted.is_null()) - .filter(lot_dsl::id.eq(id)) - .set(lot_dsl::time_deleted.eq(now)) - .execute_async(&conn) - .await?; - - diesel::delete(block_dsl::address_lot_block) - .filter(block_dsl::address_lot_id.eq(id)) - .execute_async(&conn) - .await?; - - Ok(()) - }) - .await - .map_err(|e| match e { - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - TxnError::CustomError(AddressLotDeleteError::LotInUse) => { - Error::invalid_request("lot is in use") - } - }) + diesel::delete(block_dsl::address_lot_block) + .filter(block_dsl::address_lot_id.eq(id)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + AddressLotDeleteError::LotInUse => { + Error::invalid_request("lot is in use") + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } pub async fn address_lot_list( diff --git a/nexus/db-queries/src/db/datastore/bgp.rs b/nexus/db-queries/src/db/datastore/bgp.rs index ff314a2564..28075b0ded 100644 --- a/nexus/db-queries/src/db/datastore/bgp.rs +++ b/nexus/db-queries/src/db/datastore/bgp.rs @@ -3,11 +3,11 @@ use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::Name; use crate::db::model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig}; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use nexus_types::external_api::params; @@ -30,33 +30,33 @@ impl DataStore { use db::schema::{ bgp_announce_set, bgp_announce_set::dsl as announce_set_dsl, }; - let pool = self.pool_connection_authorized(opctx).await?; - - pool.transaction_async(|conn| async move { - let id: Uuid = match &config.bgp_announce_set_id { - NameOrId::Name(name) => { - announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::time_deleted.is_null()) - .filter(bgp_announce_set::name.eq(name.to_string())) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await? - } - NameOrId::Id(id) => *id, - }; - - let config = BgpConfig::from_config_create(config, id); - - let result = diesel::insert_into(dsl::bgp_config) - .values(config.clone()) - .returning(BgpConfig::as_returning()) - .get_result_async(&conn) - .await?; - Ok(result) - }) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("bgp_config_set") + .transaction(&conn, |conn| async move { + let id: Uuid = match &config.bgp_announce_set_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::time_deleted.is_null()) + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => *id, + }; + + let config = BgpConfig::from_config_create(config, id); + + let result = diesel::insert_into(dsl::bgp_config) + .values(config.clone()) + .returning(BgpConfig::as_returning()) + .get_result_async(&conn) + .await?; + Ok(result) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn bgp_config_delete( @@ -74,54 +74,59 @@ impl DataStore { enum BgpConfigDeleteError { ConfigInUse, } - type TxnError = TransactionError; - - let pool = self.pool_connection_authorized(opctx).await?; - pool.transaction_async(|conn| async move { - let name_or_id = sel.name_or_id.clone(); - - let id: Uuid = match name_or_id { - NameOrId::Id(id) => id, - NameOrId::Name(name) => { - bgp_config_dsl::bgp_config - .filter(bgp_config::name.eq(name.to_string())) - .select(bgp_config::id) - .limit(1) - .first_async::(&conn) - .await? - } - }; - - let count = - sps_bgp_peer_config_dsl::switch_port_settings_bgp_peer_config - .filter(sps_bgp_peer_config::bgp_config_id.eq(id)) - .count() - .execute_async(&conn) - .await?; - - if count > 0 { - return Err(TxnError::CustomError( - BgpConfigDeleteError::ConfigInUse, - )); - } - diesel::update(bgp_config_dsl::bgp_config) - .filter(bgp_config_dsl::id.eq(id)) - .set(bgp_config_dsl::time_deleted.eq(Utc::now())) - .execute_async(&conn) - .await?; + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("bgp_config_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let name_or_id = sel.name_or_id.clone(); + + let id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => { + bgp_config_dsl::bgp_config + .filter(bgp_config::name.eq(name.to_string())) + .select(bgp_config::id) + .limit(1) + .first_async::(&conn) + .await? + } + }; + + let count = + sps_bgp_peer_config_dsl::switch_port_settings_bgp_peer_config + .filter(sps_bgp_peer_config::bgp_config_id.eq(id)) + .count() + .execute_async(&conn) + .await?; + + if count > 0 { + return Err(err.bail(BgpConfigDeleteError::ConfigInUse)); + } + + diesel::update(bgp_config_dsl::bgp_config) + .filter(bgp_config_dsl::id.eq(id)) + .set(bgp_config_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; - Ok(()) - }) - .await - .map_err(|e| match e { - TxnError::CustomError(BgpConfigDeleteError::ConfigInUse) => { - Error::invalid_request("BGP config in use") - } - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + BgpConfigDeleteError::ConfigInUse => { + Error::invalid_request("BGP config in use") + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } pub async fn bgp_config_get( @@ -131,7 +136,7 @@ impl DataStore { ) -> LookupResult { use db::schema::bgp_config; use db::schema::bgp_config::dsl; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let name_or_id = name_or_id.clone(); @@ -140,14 +145,14 @@ impl DataStore { .filter(bgp_config::name.eq(name.to_string())) .select(BgpConfig::as_select()) .limit(1) - .first_async::(&*pool) + .first_async::(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), NameOrId::Id(id) => dsl::bgp_config .filter(bgp_config::id.eq(id)) .select(BgpConfig::as_select()) .limit(1) - .first_async::(&*pool) + .first_async::(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), }?; @@ -162,7 +167,7 @@ impl DataStore { ) -> ListResultVec { use db::schema::bgp_config::dsl; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; match pagparams { PaginatedBy::Id(pagparams) => { @@ -176,7 +181,7 @@ impl DataStore { } .filter(dsl::time_deleted.is_null()) .select(BgpConfig::as_select()) - .load_async(&*pool) + .load_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } @@ -195,47 +200,64 @@ impl DataStore { enum BgpAnnounceListError { AnnounceSetNotFound(Name), } - type TxnError = TransactionError; - - let pool = self.pool_connection_authorized(opctx).await?; - pool.transaction_async(|conn| async move { - let name_or_id = sel.name_or_id.clone(); - - let announce_id: Uuid = match name_or_id { - NameOrId::Id(id) => id, - NameOrId::Name(name) => announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::time_deleted.is_null()) - .filter(bgp_announce_set::name.eq(name.to_string())) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError( - BgpAnnounceListError::AnnounceSetNotFound( - Name::from(name.clone()), - ), - ) - })?, - }; - - let result = announce_dsl::bgp_announcement - .filter(announce_dsl::announce_set_id.eq(announce_id)) - .select(BgpAnnouncement::as_select()) - .load_async(&conn) - .await?; - - Ok(result) - }) - .await - .map_err(|e| match e { - TxnError::CustomError( - BgpAnnounceListError::AnnounceSetNotFound(name), - ) => Error::not_found_by_name(ResourceType::BgpAnnounceSet, &name), - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("bgp_announce_list") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let name_or_id = sel.name_or_id.clone(); + + let announce_id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter( + bgp_announce_set::time_deleted.is_null(), + ) + .filter( + bgp_announce_set::name.eq(name.to_string()), + ) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or( + e, + BgpAnnounceListError::AnnounceSetNotFound( + Name::from(name.clone()), + ) + ) + })? + } + }; + + let result = announce_dsl::bgp_announcement + .filter(announce_dsl::announce_set_id.eq(announce_id)) + .select(BgpAnnouncement::as_select()) + .load_async(&conn) + .await?; + + Ok(result) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + BgpAnnounceListError::AnnounceSetNotFound(name) => { + Error::not_found_by_name( + ResourceType::BgpAnnounceSet, + &name, + ) + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } pub async fn bgp_create_announce_set( @@ -246,37 +268,39 @@ impl DataStore { use db::schema::bgp_announce_set::dsl as announce_set_dsl; use db::schema::bgp_announcement::dsl as bgp_announcement_dsl; - let pool = self.pool_connection_authorized(opctx).await?; - pool.transaction_async(|conn| async move { - let bas: BgpAnnounceSet = announce.clone().into(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("bgp_create_announce_set") + .transaction(&conn, |conn| async move { + let bas: BgpAnnounceSet = announce.clone().into(); - let db_as: BgpAnnounceSet = - diesel::insert_into(announce_set_dsl::bgp_announce_set) - .values(bas.clone()) - .returning(BgpAnnounceSet::as_returning()) - .get_result_async::(&conn) - .await?; - - let mut db_annoucements = Vec::new(); - for a in &announce.announcement { - let an = BgpAnnouncement { - announce_set_id: db_as.id(), - address_lot_block_id: bas.identity.id, - network: a.network.into(), - }; - let an = - diesel::insert_into(bgp_announcement_dsl::bgp_announcement) - .values(an.clone()) - .returning(BgpAnnouncement::as_returning()) - .get_result_async::(&conn) + let db_as: BgpAnnounceSet = + diesel::insert_into(announce_set_dsl::bgp_announce_set) + .values(bas.clone()) + .returning(BgpAnnounceSet::as_returning()) + .get_result_async::(&conn) .await?; - db_annoucements.push(an); - } - Ok((db_as, db_annoucements)) - }) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + let mut db_annoucements = Vec::new(); + for a in &announce.announcement { + let an = BgpAnnouncement { + announce_set_id: db_as.id(), + address_lot_block_id: bas.identity.id, + network: a.network.into(), + }; + let an = diesel::insert_into( + bgp_announcement_dsl::bgp_announcement, + ) + .values(an.clone()) + .returning(BgpAnnouncement::as_returning()) + .get_result_async::(&conn) + .await?; + db_annoucements.push(an); + } + + Ok((db_as, db_annoucements)) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn bgp_delete_announce_set( @@ -295,57 +319,67 @@ impl DataStore { enum BgpAnnounceSetDeleteError { AnnounceSetInUse, } - type TxnError = TransactionError; - let pool = self.pool_connection_authorized(opctx).await?; + let conn = self.pool_connection_authorized(opctx).await?; let name_or_id = sel.name_or_id.clone(); - pool.transaction_async(|conn| async move { - let id: Uuid = match name_or_id { - NameOrId::Name(name) => { - announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::name.eq(name.to_string())) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await? - } - NameOrId::Id(id) => id, - }; - - let count = bgp_config_dsl::bgp_config - .filter(bgp_config::bgp_announce_set_id.eq(id)) - .count() - .execute_async(&conn) - .await?; - - if count > 0 { - return Err(TxnError::CustomError( - BgpAnnounceSetDeleteError::AnnounceSetInUse, - )); - } + let err = OptionalError::new(); + self.transaction_retry_wrapper("bgp_delete_announce_set") + .transaction(&conn, |conn| { + let err = err.clone(); + let name_or_id = name_or_id.clone(); + async move { + let id: Uuid = match name_or_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter( + bgp_announce_set::name.eq(name.to_string()), + ) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => id, + }; + + let count = bgp_config_dsl::bgp_config + .filter(bgp_config::bgp_announce_set_id.eq(id)) + .count() + .execute_async(&conn) + .await?; - diesel::update(announce_set_dsl::bgp_announce_set) - .filter(announce_set_dsl::id.eq(id)) - .set(announce_set_dsl::time_deleted.eq(Utc::now())) - .execute_async(&conn) - .await?; + if count > 0 { + return Err(err.bail( + BgpAnnounceSetDeleteError::AnnounceSetInUse, + )); + } - diesel::delete(bgp_announcement_dsl::bgp_announcement) - .filter(bgp_announcement_dsl::announce_set_id.eq(id)) - .execute_async(&conn) - .await?; + diesel::update(announce_set_dsl::bgp_announce_set) + .filter(announce_set_dsl::id.eq(id)) + .set(announce_set_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; - Ok(()) - }) - .await - .map_err(|e| match e { - TxnError::CustomError( - BgpAnnounceSetDeleteError::AnnounceSetInUse, - ) => Error::invalid_request("BGP announce set in use"), - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + diesel::delete(bgp_announcement_dsl::bgp_announcement) + .filter(bgp_announcement_dsl::announce_set_id.eq(id)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + BgpAnnounceSetDeleteError::AnnounceSetInUse => { + Error::invalid_request("BGP announce set in use") + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } } diff --git a/nexus/db-queries/src/db/datastore/db_metadata.rs b/nexus/db-queries/src/db/datastore/db_metadata.rs index 39a70f7a1e..e579bb8476 100644 --- a/nexus/db-queries/src/db/datastore/db_metadata.rs +++ b/nexus/db-queries/src/db/datastore/db_metadata.rs @@ -8,10 +8,7 @@ use super::DataStore; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::TransactionError; -use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, -}; +use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use camino::{Utf8Path, Utf8PathBuf}; use chrono::Utc; use diesel::prelude::*; @@ -415,30 +412,30 @@ impl DataStore { target: &SemverVersion, sql: &String, ) -> Result<(), Error> { - let result = self.pool_connection_unauthorized().await?.transaction_async(|conn| async move { - if target.to_string() != EARLIEST_SUPPORTED_VERSION { - let validate_version_query = format!("SELECT CAST(\ - IF(\ - (\ - SELECT version = '{current}' and target_version = '{target}'\ - FROM omicron.public.db_metadata WHERE singleton = true\ - ),\ - 'true',\ - 'Invalid starting version for schema change'\ - ) AS BOOL\ - );"); - conn.batch_execute_async(&validate_version_query).await?; - } - conn.batch_execute_async(&sql).await?; - Ok::<_, TransactionError<()>>(()) - }).await; + let conn = self.pool_connection_unauthorized().await?; + + let result = self.transaction_retry_wrapper("apply_schema_update") + .transaction(&conn, |conn| async move { + if target.to_string() != EARLIEST_SUPPORTED_VERSION { + let validate_version_query = format!("SELECT CAST(\ + IF(\ + (\ + SELECT version = '{current}' and target_version = '{target}'\ + FROM omicron.public.db_metadata WHERE singleton = true\ + ),\ + 'true',\ + 'Invalid starting version for schema change'\ + ) AS BOOL\ + );"); + conn.batch_execute_async(&validate_version_query).await?; + } + conn.batch_execute_async(&sql).await?; + Ok(()) + }).await; match result { Ok(()) => Ok(()), - Err(TransactionError::CustomError(())) => panic!("No custom error"), - Err(TransactionError::Database(e)) => { - Err(public_error_from_diesel(e, ErrorHandler::Server)) - } + Err(e) => Err(public_error_from_diesel(e, ErrorHandler::Server)), } } diff --git a/nexus/db-queries/src/db/datastore/device_auth.rs b/nexus/db-queries/src/db/datastore/device_auth.rs index e1facb43f6..8d8e09744c 100644 --- a/nexus/db-queries/src/db/datastore/device_auth.rs +++ b/nexus/db-queries/src/db/datastore/device_auth.rs @@ -10,10 +10,8 @@ use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::DeviceAccessToken; use crate::db::model::DeviceAuthRequest; -use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use omicron_common::api::external::CreateResult; @@ -75,35 +73,40 @@ impl DataStore { RequestNotFound, TooManyRequests, } - type TxnError = TransactionError; - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - match delete_request.execute_async(&conn).await? { - 0 => { - Err(TxnError::CustomError(TokenGrantError::RequestNotFound)) + let err = crate::transaction_retry::OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("device_access_token_create") + .transaction(&conn, |conn| { + let err = err.clone(); + let insert_token = insert_token.clone(); + let delete_request = delete_request.clone(); + async move { + match delete_request.execute_async(&conn).await? { + 0 => Err(err.bail(TokenGrantError::RequestNotFound)), + 1 => Ok(insert_token.get_result_async(&conn).await?), + _ => Err(err.bail(TokenGrantError::TooManyRequests)), } - 1 => Ok(insert_token.get_result_async(&conn).await?), - _ => Err(TxnError::CustomError( - TokenGrantError::TooManyRequests, - )), } }) .await - .map_err(|e| match e { - TxnError::CustomError(TokenGrantError::RequestNotFound) => { - Error::ObjectNotFound { - type_name: ResourceType::DeviceAuthRequest, - lookup_type: LookupType::ByCompositeId( - authz_request.id(), - ), + .map_err(|e| { + if let Some(err) = err.take() { + match err { + TokenGrantError::RequestNotFound => { + Error::ObjectNotFound { + type_name: ResourceType::DeviceAuthRequest, + lookup_type: LookupType::ByCompositeId( + authz_request.id(), + ), + } + } + TokenGrantError::TooManyRequests => { + Error::internal_error("unexpectedly found multiple device auth requests for the same user code") + } } - } - TxnError::CustomError(TokenGrantError::TooManyRequests) => { - Error::internal_error("unexpectedly found multiple device auth requests for the same user code") - } - TxnError::Database(e) => { + } else { public_error_from_diesel(e, ErrorHandler::Server) } }) diff --git a/nexus/db-queries/src/db/datastore/dns.rs b/nexus/db-queries/src/db/datastore/dns.rs index f7ad97593e..cfd25d6a4f 100644 --- a/nexus/db-queries/src/db/datastore/dns.rs +++ b/nexus/db-queries/src/db/datastore/dns.rs @@ -67,7 +67,9 @@ impl DataStore { dns_group: DnsGroup, ) -> ListResultVec { let conn = self.pool_connection_authorized(opctx).await?; - self.dns_zones_list_all_on_connection(opctx, &conn, dns_group).await + Ok(self + .dns_zones_list_all_on_connection(opctx, &conn, dns_group) + .await?) } /// Variant of [`Self::dns_zones_list_all`] which may be called from a @@ -77,7 +79,7 @@ impl DataStore { opctx: &OpContext, conn: &async_bb8_diesel::Connection, dns_group: DnsGroup, - ) -> ListResultVec { + ) -> Result, TransactionError> { use db::schema::dns_zone::dsl; const LIMIT: usize = 5; @@ -88,8 +90,7 @@ impl DataStore { .limit(i64::try_from(LIMIT).unwrap()) .select(DnsZone::as_select()) .load_async(conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .await?; bail_unless!( list.len() < LIMIT, @@ -106,12 +107,14 @@ impl DataStore { opctx: &OpContext, dns_group: DnsGroup, ) -> LookupResult { - self.dns_group_latest_version_conn( - opctx, - &*self.pool_connection_authorized(opctx).await?, - dns_group, - ) - .await + let version = self + .dns_group_latest_version_conn( + opctx, + &*self.pool_connection_authorized(opctx).await?, + dns_group, + ) + .await?; + Ok(version) } pub async fn dns_group_latest_version_conn( @@ -119,7 +122,7 @@ impl DataStore { opctx: &OpContext, conn: &async_bb8_diesel::Connection, dns_group: DnsGroup, - ) -> LookupResult { + ) -> Result> { opctx.authorize(authz::Action::Read, &authz::DNS_CONFIG).await?; use db::schema::dns_version::dsl; let versions = dsl::dns_version @@ -128,8 +131,7 @@ impl DataStore { .limit(1) .select(DnsVersion::as_select()) .load_async(conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .await?; bail_unless!( versions.len() == 1, @@ -377,28 +379,17 @@ impl DataStore { opctx: &OpContext, conn: &async_bb8_diesel::Connection, update: DnsVersionUpdateBuilder, - ) -> Result<(), Error> { + ) -> Result<(), TransactionError> { opctx.authorize(authz::Action::Modify, &authz::DNS_CONFIG).await?; let zones = self .dns_zones_list_all_on_connection(opctx, conn, update.dns_group) .await?; - let result = conn - .transaction_async(|c| async move { - self.dns_update_internal(opctx, &c, update, zones) - .await - .map_err(TransactionError::CustomError) - }) - .await; - - match result { - Ok(()) => Ok(()), - Err(TransactionError::CustomError(e)) => Err(e), - Err(TransactionError::Database(e)) => { - Err(public_error_from_diesel(e, ErrorHandler::Server)) - } - } + conn.transaction_async(|c| async move { + self.dns_update_internal(opctx, &c, update, zones).await + }) + .await } // This must only be used inside a transaction. Otherwise, it may make @@ -409,7 +400,7 @@ impl DataStore { conn: &async_bb8_diesel::Connection, update: DnsVersionUpdateBuilder, zones: Vec, - ) -> Result<(), Error> { + ) -> Result<(), TransactionError> { // TODO-scalability TODO-performance This would be much better as a CTE // for all the usual reasons described in RFD 192. Using an interactive // transaction here means that either we wind up holding database locks @@ -455,10 +446,7 @@ impl DataStore { diesel::insert_into(dsl::dns_version) .values(new_version) .execute_async(conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - })?; + .await?; } { @@ -480,8 +468,7 @@ impl DataStore { ) .set(dsl::version_removed.eq(new_version_num)) .execute_async(conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .await?; bail_unless!( nremoved == ntoremove, @@ -495,10 +482,7 @@ impl DataStore { let nadded = diesel::insert_into(dsl::dns_name) .values(new_names) .execute_async(conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - })?; + .await?; bail_unless!( nadded == ntoadd, @@ -1684,6 +1668,10 @@ mod test { let conn = datastore.pool_connection_for_tests().await.unwrap(); let error = datastore.dns_update(&opctx, &conn, update).await.unwrap_err(); + let error = match error { + TransactionError::CustomError(err) => err, + _ => panic!("Unexpected error: {:?}", error), + }; assert_eq!( error.to_string(), "Internal Error: updated wrong number of dns_name \ @@ -1707,11 +1695,15 @@ mod test { update.add_name(String::from("n2"), records1.clone()).unwrap(); let conn = datastore.pool_connection_for_tests().await.unwrap(); - let error = - datastore.dns_update(&opctx, &conn, update).await.unwrap_err(); + let error = Error::from( + datastore.dns_update(&opctx, &conn, update).await.unwrap_err(), + ); let msg = error.to_string(); - assert!(msg.starts_with("Internal Error: ")); - assert!(msg.contains("violates unique constraint")); + assert!(msg.starts_with("Internal Error: "), "Message: {msg:}"); + assert!( + msg.contains("violates unique constraint"), + "Message: {msg:}" + ); } let dns_config = datastore diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs index e663130a84..4e34bfc15c 100644 --- a/nexus/db-queries/src/db/datastore/external_ip.rs +++ b/nexus/db-queries/src/db/datastore/external_ip.rs @@ -10,7 +10,9 @@ use crate::authz::ApiResource; use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; +use crate::db::error::TransactionError; use crate::db::lookup::LookupPath; use crate::db::model::ExternalIp; use crate::db::model::IncompleteExternalIp; @@ -132,7 +134,8 @@ impl DataStore { data: IncompleteExternalIp, ) -> CreateResult { let conn = self.pool_connection_authorized(opctx).await?; - Self::allocate_external_ip_on_connection(&conn, data).await + let ip = Self::allocate_external_ip_on_connection(&conn, data).await?; + Ok(ip) } /// Variant of [Self::allocate_external_ip] which may be called from a @@ -140,23 +143,30 @@ impl DataStore { pub(crate) async fn allocate_external_ip_on_connection( conn: &async_bb8_diesel::Connection, data: IncompleteExternalIp, - ) -> CreateResult { + ) -> Result> { let explicit_ip = data.explicit_ip().is_some(); NextExternalIp::new(data).get_result_async(conn).await.map_err(|e| { use diesel::result::Error::NotFound; match e { NotFound => { if explicit_ip { - Error::invalid_request( + TransactionError::CustomError(Error::invalid_request( "Requested external IP address not available", - ) + )) } else { - Error::invalid_request( + TransactionError::CustomError(Error::invalid_request( "No external IP addresses available", - ) + )) + } + } + _ => { + if retryable(&e) { + return TransactionError::Database(e); } + TransactionError::CustomError( + crate::db::queries::external_ip::from_diesel(e), + ) } - _ => crate::db::queries::external_ip::from_diesel(e), } }) } diff --git a/nexus/db-queries/src/db/datastore/identity_provider.rs b/nexus/db-queries/src/db/datastore/identity_provider.rs index fdc9a020e7..cee577acd6 100644 --- a/nexus/db-queries/src/db/datastore/identity_provider.rs +++ b/nexus/db-queries/src/db/datastore/identity_provider.rs @@ -14,7 +14,6 @@ use crate::db::identity::Resource; use crate::db::model::IdentityProvider; use crate::db::model::Name; use crate::db::pagination::paginated; -use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use omicron_common::api::external::http_pagination::PaginatedBy; @@ -63,36 +62,47 @@ impl DataStore { assert_eq!(provider.silo_id, authz_idp_list.silo().id()); let name = provider.identity().name.to_string(); - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - // insert silo identity provider record with type Saml - use db::schema::identity_provider::dsl as idp_dsl; - diesel::insert_into(idp_dsl::identity_provider) - .values(db::model::IdentityProvider { - identity: db::model::IdentityProviderIdentity { - id: provider.identity.id, - name: provider.identity.name.clone(), - description: provider.identity.description.clone(), - time_created: provider.identity.time_created, - time_modified: provider.identity.time_modified, - time_deleted: provider.identity.time_deleted, - }, - silo_id: provider.silo_id, - provider_type: db::model::IdentityProviderType::Saml, - }) - .execute_async(&conn) - .await?; + let conn = self.pool_connection_authorized(opctx).await?; - // insert silo saml identity provider record - use db::schema::saml_identity_provider::dsl; - let result = diesel::insert_into(dsl::saml_identity_provider) - .values(provider) - .returning(db::model::SamlIdentityProvider::as_returning()) - .get_result_async(&conn) - .await?; + self.transaction_retry_wrapper("saml_identity_provider_create") + .transaction(&conn, |conn| { + let provider = provider.clone(); + async move { + // insert silo identity provider record with type Saml + use db::schema::identity_provider::dsl as idp_dsl; + diesel::insert_into(idp_dsl::identity_provider) + .values(db::model::IdentityProvider { + identity: db::model::IdentityProviderIdentity { + id: provider.identity.id, + name: provider.identity.name.clone(), + description: provider + .identity + .description + .clone(), + time_created: provider.identity.time_created, + time_modified: provider.identity.time_modified, + time_deleted: provider.identity.time_deleted, + }, + silo_id: provider.silo_id, + provider_type: + db::model::IdentityProviderType::Saml, + }) + .execute_async(&conn) + .await?; - Ok(result) + // insert silo saml identity provider record + use db::schema::saml_identity_provider::dsl; + let result = + diesel::insert_into(dsl::saml_identity_provider) + .values(provider) + .returning( + db::model::SamlIdentityProvider::as_returning(), + ) + .get_result_async(&conn) + .await?; + + Ok(result) + } }) .await .map_err(|e| { diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index 44cd7a95b7..2e7f9da5b7 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -148,6 +148,7 @@ pub type DataStoreConnection<'a> = pub struct DataStore { pool: Arc, virtual_provisioning_collection_producer: crate::provisioning::Producer, + transaction_retry_producer: crate::transaction_retry::Producer, } // The majority of `DataStore`'s methods live in our submodules as a concession @@ -164,6 +165,8 @@ impl DataStore { pool, virtual_provisioning_collection_producer: crate::provisioning::Producer::new(), + transaction_retry_producer: crate::transaction_retry::Producer::new( + ), }; Ok(datastore) } @@ -210,6 +213,29 @@ impl DataStore { self.virtual_provisioning_collection_producer.clone(), ) .unwrap(); + registry + .register_producer(self.transaction_retry_producer.clone()) + .unwrap(); + } + + /// Constructs a transaction retry helper + /// + /// Automatically wraps the underlying producer + pub fn transaction_retry_wrapper( + &self, + name: &'static str, + ) -> crate::transaction_retry::RetryHelper { + crate::transaction_retry::RetryHelper::new( + &self.transaction_retry_producer, + name, + ) + } + + #[cfg(test)] + pub(crate) fn transaction_retry_producer( + &self, + ) -> &crate::transaction_retry::Producer { + &self.transaction_retry_producer } /// Returns a connection to a connection from the database connection pool. diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index 06550e9439..4d4e43c9a7 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -13,7 +13,6 @@ use crate::db::collection_insert::DatastoreCollection; use crate::db::cte_utils::BoxedQuery; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::IncompleteNetworkInterface; use crate::db::model::Instance; use crate::db::model::InstanceNetworkInterface; @@ -25,7 +24,7 @@ use crate::db::model::VpcSubnet; use crate::db::pagination::paginated; use crate::db::pool::DbConnection; use crate::db::queries::network_interface; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; @@ -466,77 +465,91 @@ impl DataStore { InstanceNotStopped, FailedToUnsetPrimary(DieselError), } - type TxnError = TransactionError; + + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; if primary { - conn.transaction_async(|conn| async move { - let instance_runtime = - instance_query.get_result_async(&conn).await?.runtime_state; - if instance_runtime.propolis_id.is_some() - || instance_runtime.nexus_state != stopped - { - return Err(TxnError::CustomError( - NetworkInterfaceUpdateError::InstanceNotStopped, - )); - } + self.transaction_retry_wrapper("instance_update_network_interface") + .transaction(&conn, |conn| { + let err = err.clone(); + let stopped = stopped.clone(); + let update_target_query = update_target_query.clone(); + async move { + let instance_runtime = + instance_query.get_result_async(&conn).await?.runtime_state; + if instance_runtime.propolis_id.is_some() + || instance_runtime.nexus_state != stopped + { + return Err(err.bail(NetworkInterfaceUpdateError::InstanceNotStopped)); + } - // First, get the primary interface - let primary_interface = - find_primary_query.get_result_async(&conn).await?; - // If the target and primary are different, we need to toggle - // the primary into a secondary. - if primary_interface.identity.id != interface_id { - use crate::db::schema::network_interface::dsl; - if let Err(e) = diesel::update(dsl::network_interface) - .filter(dsl::id.eq(primary_interface.identity.id)) - .filter(dsl::kind.eq(NetworkInterfaceKind::Instance)) - .filter(dsl::time_deleted.is_null()) - .set(dsl::is_primary.eq(false)) - .execute_async(&conn) - .await - { - return Err(TxnError::CustomError( - NetworkInterfaceUpdateError::FailedToUnsetPrimary( - e, - ), - )); - } - } + // First, get the primary interface + let primary_interface = + find_primary_query.get_result_async(&conn).await?; + // If the target and primary are different, we need to toggle + // the primary into a secondary. + if primary_interface.identity.id != interface_id { + use crate::db::schema::network_interface::dsl; + if let Err(e) = diesel::update(dsl::network_interface) + .filter(dsl::id.eq(primary_interface.identity.id)) + .filter(dsl::kind.eq(NetworkInterfaceKind::Instance)) + .filter(dsl::time_deleted.is_null()) + .set(dsl::is_primary.eq(false)) + .execute_async(&conn) + .await + { + return Err(err.bail_retryable_or_else( + e, + |e| NetworkInterfaceUpdateError::FailedToUnsetPrimary(e) + )); + } + } - // In any case, update the actual target - Ok(update_target_query.get_result_async(&conn).await?) - }) + // In any case, update the actual target + update_target_query.get_result_async(&conn).await + } + }).await } else { // In this case, we can just directly apply the updates. By // construction, `updates.primary` is `None`, so nothing will // be done there. The other columns always need to be updated, and // we're only hitting a single row. Note that we still need to // verify the instance is stopped. - conn.transaction_async(|conn| async move { - let instance_state = - instance_query.get_result_async(&conn).await?.runtime_state; - if instance_state.propolis_id.is_some() - || instance_state.nexus_state != stopped - { - return Err(TxnError::CustomError( - NetworkInterfaceUpdateError::InstanceNotStopped, - )); - } - Ok(update_target_query.get_result_async(&conn).await?) - }) + self.transaction_retry_wrapper("instance_update_network_interface") + .transaction(&conn, |conn| { + let err = err.clone(); + let stopped = stopped.clone(); + let update_target_query = update_target_query.clone(); + async move { + let instance_state = + instance_query.get_result_async(&conn).await?.runtime_state; + if instance_state.propolis_id.is_some() + || instance_state.nexus_state != stopped + { + return Err(err.bail(NetworkInterfaceUpdateError::InstanceNotStopped)); + } + update_target_query.get_result_async(&conn).await + } + }).await } - .await // Convert to `InstanceNetworkInterface` before returning, we know // this is valid as we've filtered appropriately above. .map(NetworkInterface::as_instance) - .map_err(|e| match e { - TxnError::CustomError( - NetworkInterfaceUpdateError::InstanceNotStopped, - ) => Error::invalid_request( - "Instance must be stopped to update its network interfaces", - ), - _ => Error::internal_error(&format!("Transaction error: {:?}", e)), + .map_err(|e| { + if let Some(err) = err.take() { + match err { + NetworkInterfaceUpdateError::InstanceNotStopped => { + return Error::invalid_request( + "Instance must be stopped to update its network interfaces", + ); + }, + NetworkInterfaceUpdateError::FailedToUnsetPrimary(err) => { + return public_error_from_diesel(err, ErrorHandler::Server); + }, + } + } + public_error_from_diesel(e, ErrorHandler::Server) }) } } diff --git a/nexus/db-queries/src/db/datastore/project.rs b/nexus/db-queries/src/db/datastore/project.rs index c447b5bf98..ba0c64abfd 100644 --- a/nexus/db-queries/src/db/datastore/project.rs +++ b/nexus/db-queries/src/db/datastore/project.rs @@ -13,7 +13,6 @@ use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::fixed_data::project::SERVICES_PROJECT; use crate::db::fixed_data::silo::INTERNAL_SILO_ID; use crate::db::identity::Resource; @@ -24,7 +23,8 @@ use crate::db::model::ProjectUpdate; use crate::db::model::Silo; use crate::db::model::VirtualProvisioningCollection; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use omicron_common::api::external::http_pagination::PaginatedBy; @@ -151,49 +151,62 @@ impl DataStore { use db::schema::project::dsl; + let err = OptionalError::new(); let name = project.name().as_str().to_string(); + let conn = self.pool_connection_authorized(opctx).await?; + let db_project = self - .pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - let project: Project = Silo::insert_resource( - silo_id, - diesel::insert_into(dsl::project).values(project), - ) - .insert_and_get_result_async(&conn) - .await - .map_err(|e| match e { - AsyncInsertError::CollectionNotFound => { - authz_silo_inner.not_found() - } - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Project, - &name, + .transaction_retry_wrapper("project_create_in_silo") + .transaction(&conn, |conn| { + let err = err.clone(); + + let authz_silo_inner = authz_silo_inner.clone(); + let name = name.clone(); + let project = project.clone(); + async move { + let project: Project = Silo::insert_resource( + silo_id, + diesel::insert_into(dsl::project).values(project), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => { + err.bail(authz_silo_inner.not_found()) + } + AsyncInsertError::DatabaseError(diesel_error) => err + .bail_retryable_or_else( + diesel_error, + |diesel_error| { + public_error_from_diesel( + diesel_error, + ErrorHandler::Conflict( + ResourceType::Project, + &name, + ), + ) + }, ), - ) - } - })?; - - // Create resource provisioning for the project. - self.virtual_provisioning_collection_create_on_connection( - &conn, - VirtualProvisioningCollection::new( - project.id(), - CollectionTypeProvisioned::Project, - ), - ) - .await?; - Ok(project) + })?; + + // Create resource provisioning for the project. + self.virtual_provisioning_collection_create_on_connection( + &conn, + VirtualProvisioningCollection::new( + project.id(), + CollectionTypeProvisioned::Project, + ), + ) + .await?; + Ok(project) + } }) .await - .map_err(|e| match e { - TransactionError::CustomError(e) => e, - TransactionError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + .map_err(|e| { + if let Some(err) = err.take() { + return err; } + public_error_from_diesel(e, ErrorHandler::Server) })?; Ok(( @@ -230,47 +243,56 @@ impl DataStore { use db::schema::project::dsl; - type TxnError = TransactionError; - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - let now = Utc::now(); - let updated_rows = diesel::update(dsl::project) - .filter(dsl::time_deleted.is_null()) - .filter(dsl::id.eq(authz_project.id())) - .filter(dsl::rcgen.eq(db_project.rcgen)) - .set(dsl::time_deleted.eq(now)) - .returning(Project::as_returning()) - .execute_async(&conn) - .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_project), - ) - })?; + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("project_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let now = Utc::now(); + let updated_rows = diesel::update(dsl::project) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_project.id())) + .filter(dsl::rcgen.eq(db_project.rcgen)) + .set(dsl::time_deleted.eq(now)) + .returning(Project::as_returning()) + .execute_async(&conn) + .await + .map_err(|e| { + err.bail_retryable_or_else(e, |e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource( + authz_project, + ), + ) + }) + })?; + + if updated_rows == 0 { + return Err(err.bail(Error::InvalidRequest { + message: + "deletion failed due to concurrent modification" + .to_string(), + })); + } - if updated_rows == 0 { - return Err(TxnError::CustomError(Error::InvalidRequest { - message: - "deletion failed due to concurrent modification" - .to_string(), - })); + self.virtual_provisioning_collection_delete_on_connection( + &opctx.log, + &conn, + db_project.id(), + ) + .await?; + Ok(()) } - - self.virtual_provisioning_collection_delete_on_connection( - &conn, - db_project.id(), - ) - .await?; - Ok(()) }) .await - .map_err(|e| match e { - TxnError::CustomError(e) => e, - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + .map_err(|e| { + if let Some(err) = err.take() { + return err; } + public_error_from_diesel(e, ErrorHandler::Server) })?; Ok(()) } diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index e11377f11a..a69386cfd0 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -13,8 +13,9 @@ use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; +use crate::db::error::MaybeRetryable::*; use crate::db::fixed_data::silo::INTERNAL_SILO_ID; use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET; use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; @@ -26,6 +27,7 @@ use crate::db::model::Rack; use crate::db::model::Zpool; use crate::db::pagination::paginated; use crate::db::pool::DbConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; @@ -58,6 +60,7 @@ use omicron_common::api::external::ResourceType; use omicron_common::api::external::UpdateResult; use omicron_common::bail_unless; use std::net::IpAddr; +use std::sync::{Arc, OnceLock}; use uuid::Uuid; /// Groups arguments related to rack initialization @@ -88,18 +91,30 @@ enum RackInitError { DnsSerialization(Error), Silo(Error), RoleAssignment(Error), + // Retryable database error + Retryable(DieselError), + // Other non-retryable database error + Database(DieselError), } -type TxnError = TransactionError; -impl From for Error { - fn from(e: TxnError) -> Self { +// Catch-all for Diesel error conversion into RackInitError, which +// can also label errors as retryable. +impl From for RackInitError { + fn from(e: DieselError) -> Self { + if retryable(&e) { + Self::Retryable(e) + } else { + Self::Database(e) + } + } +} + +impl From for Error { + fn from(e: RackInitError) -> Self { match e { - TxnError::CustomError(RackInitError::AddingIp(err)) => err, - TxnError::CustomError(RackInitError::AddingNic(err)) => err, - TxnError::CustomError(RackInitError::DatasetInsert { - err, - zpool_id, - }) => match err { + RackInitError::AddingIp(err) => err, + RackInitError::AddingNic(err) => err, + RackInitError::DatasetInsert { err, zpool_id } => match err { AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { type_name: ResourceType::Zpool, lookup_type: LookupType::ById(zpool_id), @@ -108,43 +123,36 @@ impl From for Error { public_error_from_diesel(e, ErrorHandler::Server) } }, - TxnError::CustomError(RackInitError::ServiceInsert(err)) => { - Error::internal_error(&format!( - "failed to insert Service record: {:#}", - err - )) - } - TxnError::CustomError(RackInitError::RackUpdate { - err, - rack_id, - }) => public_error_from_diesel( - err, - ErrorHandler::NotFoundByLookup( - ResourceType::Rack, - LookupType::ById(rack_id), - ), + RackInitError::ServiceInsert(err) => Error::internal_error( + &format!("failed to insert Service record: {:#}", err), ), - TxnError::CustomError(RackInitError::DnsSerialization(err)) => { - Error::internal_error(&format!( - "failed to serialize initial DNS records: {:#}", - err - )) - } - TxnError::CustomError(RackInitError::Silo(err)) => { - Error::internal_error(&format!( - "failed to create recovery Silo: {:#}", - err - )) - } - TxnError::CustomError(RackInitError::RoleAssignment(err)) => { - Error::internal_error(&format!( - "failed to assign role to initial user: {:#}", - err - )) - } - TxnError::Database(e) => { - Error::internal_error(&format!("Transaction error: {}", e)) + RackInitError::RackUpdate { err, rack_id } => { + public_error_from_diesel( + err, + ErrorHandler::NotFoundByLookup( + ResourceType::Rack, + LookupType::ById(rack_id), + ), + ) } + RackInitError::DnsSerialization(err) => Error::internal_error( + &format!("failed to serialize initial DNS records: {:#}", err), + ), + RackInitError::Silo(err) => Error::internal_error(&format!( + "failed to create recovery Silo: {:#}", + err + )), + RackInitError::RoleAssignment(err) => Error::internal_error( + &format!("failed to assign role to initial user: {:#}", err), + ), + RackInitError::Retryable(err) => Error::internal_error(&format!( + "failed operation due to database contention: {:#}", + err + )), + RackInitError::Database(err) => Error::internal_error(&format!( + "failed operation due to database error: {:#}", + err + )), } } } @@ -336,9 +344,6 @@ impl DataStore { Ok(()) } - // The following methods which return a `TxnError` take a `conn` parameter - // which comes from the transaction created in `rack_set_initialized`. - #[allow(clippy::too_many_arguments)] async fn rack_create_recovery_silo( &self, @@ -350,7 +355,7 @@ impl DataStore { recovery_user_id: external_params::UserId, recovery_user_password_hash: omicron_passwords::PasswordHashString, dns_update: DnsVersionUpdateBuilder, - ) -> Result<(), TxnError> { + ) -> Result<(), RackInitError> { let db_silo = self .silo_create_conn( conn, @@ -361,8 +366,10 @@ impl DataStore { dns_update, ) .await - .map_err(RackInitError::Silo) - .map_err(TxnError::CustomError)?; + .map_err(|err| match err.retryable() { + NotRetryable(err) => RackInitError::Silo(err.into()), + Retryable(err) => RackInitError::Retryable(err), + })?; info!(log, "Created recovery silo"); // Create the first user in the initial Recovery Silo @@ -416,8 +423,7 @@ impl DataStore { }], ) .await - .map_err(RackInitError::RoleAssignment) - .map_err(TxnError::CustomError)?; + .map_err(RackInitError::RoleAssignment)?; debug!(log, "Generated role assignment queries"); q1.execute_async(conn).await?; @@ -433,7 +439,7 @@ impl DataStore { log: &slog::Logger, service_pool: &db::model::IpPool, service: internal_params::ServicePutRequest, - ) -> Result<(), TxnError> { + ) -> Result<(), RackInitError> { use internal_params::ServiceKind; let service_db = db::model::Service::new( @@ -443,9 +449,12 @@ impl DataStore { service.address, service.kind.clone().into(), ); - self.service_upsert_conn(conn, service_db).await.map_err(|e| { - TxnError::CustomError(RackInitError::ServiceInsert(e)) - })?; + self.service_upsert_conn(conn, service_db).await.map_err( + |e| match e.retryable() { + Retryable(e) => RackInitError::Retryable(e), + NotRetryable(e) => RackInitError::ServiceInsert(e.into()), + }, + )?; // For services with external connectivity, we record their // explicit IP allocation and create a service NIC as well. @@ -476,9 +485,7 @@ impl DataStore { Some(nic.ip), Some(nic.mac), ) - .map_err(|e| { - TxnError::CustomError(RackInitError::AddingNic(e)) - })?; + .map_err(|e| RackInitError::AddingNic(e))?; Some((db_ip, db_nic)) } ServiceKind::BoundaryNtp { snat, ref nic } => { @@ -500,9 +507,7 @@ impl DataStore { Some(nic.ip), Some(nic.mac), ) - .map_err(|e| { - TxnError::CustomError(RackInitError::AddingNic(e)) - })?; + .map_err(|e| RackInitError::AddingNic(e))?; Some((db_ip, db_nic)) } _ => None, @@ -517,7 +522,10 @@ impl DataStore { IP address for {}", service.kind, ); - TxnError::CustomError(RackInitError::AddingIp(err)) + match err.retryable() { + Retryable(e) => RackInitError::Retryable(e), + NotRetryable(e) => RackInitError::AddingIp(e.into()), + } })?; self.create_network_interface_raw_conn(conn, db_nic) @@ -530,9 +538,10 @@ impl DataStore { _, db::model::NetworkInterfaceKind::Service, ) => Ok(()), - _ => Err(TxnError::CustomError( - RackInitError::AddingNic(e.into_external()), - )), + InsertError::Retryable(err) => { + Err(RackInitError::Retryable(err)) + } + _ => Err(RackInitError::AddingNic(e.into_external())), } })?; } @@ -551,146 +560,187 @@ impl DataStore { opctx.authorize(authz::Action::CreateChild, &authz::FLEET).await?; - let rack_id = rack_init.rack_id; - let services = rack_init.services; - let datasets = rack_init.datasets; - let service_ip_pool_ranges = rack_init.service_ip_pool_ranges; - let internal_dns = rack_init.internal_dns; - let external_dns = rack_init.external_dns; - let (authz_service_pool, service_pool) = self.ip_pools_service_lookup(&opctx).await?; // NOTE: This operation could likely be optimized with a CTE, but given // the low-frequency of calls, this optimization has been deferred. let log = opctx.log.clone(); + let err = Arc::new(OnceLock::new()); + + // NOTE: This transaction cannot yet be made retryable, as it uses + // nested transactions. let rack = self .pool_connection_authorized(opctx) .await? - .transaction_async(|conn| async move { - // Early exit if the rack has already been initialized. - let rack = rack_dsl::rack - .filter(rack_dsl::id.eq(rack_id)) - .select(Rack::as_select()) - .get_result_async(&conn) - .await - .map_err(|e| { - warn!(log, "Initializing Rack: Rack UUID not found"); - TxnError::CustomError(RackInitError::RackUpdate { - err: e, - rack_id, - }) - })?; - if rack.initialized { - info!(log, "Early exit: Rack already initialized"); - return Ok(rack); - } + .transaction_async(|conn| { + let err = err.clone(); + let log = log.clone(); + let authz_service_pool = authz_service_pool.clone(); + let rack_init = rack_init.clone(); + let service_pool = service_pool.clone(); + async move { + let rack_id = rack_init.rack_id; + let services = rack_init.services; + let datasets = rack_init.datasets; + let service_ip_pool_ranges = rack_init.service_ip_pool_ranges; + let internal_dns = rack_init.internal_dns; + let external_dns = rack_init.external_dns; + + // Early exit if the rack has already been initialized. + let rack = rack_dsl::rack + .filter(rack_dsl::id.eq(rack_id)) + .select(Rack::as_select()) + .get_result_async(&conn) + .await + .map_err(|e| { + warn!(log, "Initializing Rack: Rack UUID not found"); + err.set(RackInitError::RackUpdate { + err: e, + rack_id, + }).unwrap(); + DieselError::RollbackTransaction + })?; + if rack.initialized { + info!(log, "Early exit: Rack already initialized"); + return Ok::<_, DieselError>(rack); + } - // Otherwise, insert services and datasets. + // Otherwise, insert services and datasets. - // Set up the IP pool for internal services. - for range in service_ip_pool_ranges { - Self::ip_pool_add_range_on_connection( - &conn, - opctx, - &authz_service_pool, - &range, - ) - .await - .map_err(|err| { - warn!( - log, - "Initializing Rack: Failed to add IP pool range" - ); - TxnError::CustomError(RackInitError::AddingIp(err)) - })?; - } + // Set up the IP pool for internal services. + for range in service_ip_pool_ranges { + Self::ip_pool_add_range_on_connection( + &conn, + opctx, + &authz_service_pool, + &range, + ) + .await + .map_err(|e| { + warn!( + log, + "Initializing Rack: Failed to add IP pool range" + ); + err.set(RackInitError::AddingIp(e)).unwrap(); + DieselError::RollbackTransaction + })?; + } + + // Allocate records for all services. + for service in services { + self.rack_populate_service_records( + &conn, + &log, + &service_pool, + service, + ) + .await + .map_err(|e| { + err.set(e).unwrap(); + DieselError::RollbackTransaction + })?; + } + info!(log, "Inserted services"); + + for dataset in datasets { + use db::schema::dataset::dsl; + let zpool_id = dataset.pool_id; + >::insert_resource( + zpool_id, + diesel::insert_into(dsl::dataset) + .values(dataset.clone()) + .on_conflict(dsl::id) + .do_update() + .set(( + dsl::time_modified.eq(Utc::now()), + dsl::pool_id.eq(excluded(dsl::pool_id)), + dsl::ip.eq(excluded(dsl::ip)), + dsl::port.eq(excluded(dsl::port)), + dsl::kind.eq(excluded(dsl::kind)), + )), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| { + err.set(RackInitError::DatasetInsert { + err: e, + zpool_id, + }).unwrap(); + DieselError::RollbackTransaction + })?; + } + info!(log, "Inserted datasets"); - // Allocate records for all services. - for service in services { - self.rack_populate_service_records( + // Insert the initial contents of the internal and external DNS + // zones. + Self::load_dns_data(&conn, internal_dns) + .await + .map_err(|e| { + err.set(RackInitError::DnsSerialization(e)).unwrap(); + DieselError::RollbackTransaction + })?; + info!(log, "Populated DNS tables for internal DNS"); + + Self::load_dns_data(&conn, external_dns) + .await + .map_err(|e| { + err.set(RackInitError::DnsSerialization(e)).unwrap(); + DieselError::RollbackTransaction + })?; + info!(log, "Populated DNS tables for external DNS"); + + // Create the initial Recovery Silo + self.rack_create_recovery_silo( + &opctx, &conn, &log, - &service_pool, - service, - ) - .await?; - } - info!(log, "Inserted services"); - - for dataset in datasets { - use db::schema::dataset::dsl; - let zpool_id = dataset.pool_id; - >::insert_resource( - zpool_id, - diesel::insert_into(dsl::dataset) - .values(dataset.clone()) - .on_conflict(dsl::id) - .do_update() - .set(( - dsl::time_modified.eq(Utc::now()), - dsl::pool_id.eq(excluded(dsl::pool_id)), - dsl::ip.eq(excluded(dsl::ip)), - dsl::port.eq(excluded(dsl::port)), - dsl::kind.eq(excluded(dsl::kind)), - )), + rack_init.recovery_silo, + rack_init.recovery_silo_fq_dns_name, + rack_init.recovery_user_id, + rack_init.recovery_user_password_hash, + rack_init.dns_update, ) - .insert_and_get_result_async(&conn) .await - .map_err(|err| { - TxnError::CustomError(RackInitError::DatasetInsert { - err, - zpool_id, - }) + .map_err(|e| match e { + RackInitError::Retryable(e) => e, + _ => { + err.set(e).unwrap(); + DieselError::RollbackTransaction + }, })?; - } - info!(log, "Inserted datasets"); - - // Insert the initial contents of the internal and external DNS - // zones. - Self::load_dns_data(&conn, internal_dns) - .await - .map_err(RackInitError::DnsSerialization) - .map_err(TxnError::CustomError)?; - info!(log, "Populated DNS tables for internal DNS"); - Self::load_dns_data(&conn, external_dns) - .await - .map_err(RackInitError::DnsSerialization) - .map_err(TxnError::CustomError)?; - info!(log, "Populated DNS tables for external DNS"); - - // Create the initial Recovery Silo - self.rack_create_recovery_silo( - &opctx, - &conn, - &log, - rack_init.recovery_silo, - rack_init.recovery_silo_fq_dns_name, - rack_init.recovery_user_id, - rack_init.recovery_user_password_hash, - rack_init.dns_update, - ) - .await?; - - let rack = diesel::update(rack_dsl::rack) - .filter(rack_dsl::id.eq(rack_id)) - .set(( - rack_dsl::initialized.eq(true), - rack_dsl::time_modified.eq(Utc::now()), - )) - .returning(Rack::as_returning()) - .get_result_async::(&conn) - .await - .map_err(|err| { - TxnError::CustomError(RackInitError::RackUpdate { - err, - rack_id, - }) - })?; - Ok::<_, TxnError>(rack) - }) - .await?; + let rack = diesel::update(rack_dsl::rack) + .filter(rack_dsl::id.eq(rack_id)) + .set(( + rack_dsl::initialized.eq(true), + rack_dsl::time_modified.eq(Utc::now()), + )) + .returning(Rack::as_returning()) + .get_result_async::(&conn) + .await + .map_err(|e| { + if retryable(&e) { + return e; + } + err.set(RackInitError::RackUpdate { + err: e, + rack_id, + }).unwrap(); + DieselError::RollbackTransaction + })?; + Ok(rack) + } + }, + ) + .await + .map_err(|e| { + if let Some(err) = Arc::try_unwrap(err).unwrap().take() { + err.into() + } else { + Error::internal_error(&format!("Transaction error: {}", e)) + } + })?; Ok(rack) } @@ -745,42 +795,54 @@ impl DataStore { use crate::db::schema::external_ip::dsl as extip_dsl; use crate::db::schema::service::dsl as service_dsl; - type TxnError = TransactionError; - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - let ips = extip_dsl::external_ip - .inner_join( - service_dsl::service.on(service_dsl::id - .eq(extip_dsl::parent_id.assume_not_null())), - ) - .filter(extip_dsl::parent_id.is_not_null()) - .filter(extip_dsl::time_deleted.is_null()) - .filter(extip_dsl::is_service) - .filter(service_dsl::kind.eq(db::model::ServiceKind::Nexus)) - .select(ExternalIp::as_select()) - .get_results_async(&conn) - .await? - .into_iter() - .map(|external_ip| external_ip.ip.ip()) - .collect(); - - let dns_zones = self - .dns_zones_list_all_on_connection( - opctx, - &conn, - DnsGroup::External, - ) - .await?; - Ok((ips, dns_zones)) + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("nexus_external_addresses") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let ips = extip_dsl::external_ip + .inner_join( + service_dsl::service.on(service_dsl::id + .eq(extip_dsl::parent_id.assume_not_null())), + ) + .filter(extip_dsl::parent_id.is_not_null()) + .filter(extip_dsl::time_deleted.is_null()) + .filter(extip_dsl::is_service) + .filter( + service_dsl::kind.eq(db::model::ServiceKind::Nexus), + ) + .select(ExternalIp::as_select()) + .get_results_async(&conn) + .await? + .into_iter() + .map(|external_ip| external_ip.ip.ip()) + .collect(); + + let dns_zones = self + .dns_zones_list_all_on_connection( + opctx, + &conn, + DnsGroup::External, + ) + .await + .map_err(|e| match e.retryable() { + NotRetryable(not_retryable_err) => { + err.bail(not_retryable_err) + } + Retryable(retryable_err) => retryable_err, + })?; + + Ok((ips, dns_zones)) + } }) .await - .map_err(|error: TxnError| match error { - TransactionError::CustomError(err) => err, - TransactionError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + .map_err(|e| { + if let Some(err) = err.take() { + return err.into(); } + public_error_from_diesel(e, ErrorHandler::Server) }) } } @@ -1014,14 +1076,16 @@ mod test { async fn [](db: &DataStore) -> Vec<$model> { use crate::db::schema::$table::dsl; use nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL; - db.pool_connection_for_tests() + let conn = db.pool_connection_for_tests() .await - .unwrap() - .transaction_async(|conn| async move { + .unwrap(); + + db.transaction_retry_wrapper(concat!("fn_to_get_all_", stringify!($table))) + .transaction(&conn, |conn| async move { conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL) .await .unwrap(); - Ok::<_, crate::db::TransactionError<()>>( + Ok( dsl::$table .select($model::as_select()) .get_results_async(&conn) diff --git a/nexus/db-queries/src/db/datastore/region.rs b/nexus/db-queries/src/db/datastore/region.rs index 9465fe2792..b055a3e85c 100644 --- a/nexus/db-queries/src/db/datastore/region.rs +++ b/nexus/db-queries/src/db/datastore/region.rs @@ -10,18 +10,16 @@ use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::lookup::LookupPath; use crate::db::model::Dataset; use crate::db::model::Region; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use nexus_types::external_api::params; use omicron_common::api::external; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; -use omicron_common::backoff::{self, BackoffError}; use omicron_common::nexus_config::RegionAllocationStrategy; use slog::Logger; use uuid::Uuid; @@ -152,7 +150,7 @@ impl DataStore { /// Also updates the storage usage on their corresponding datasets. pub async fn regions_hard_delete( &self, - log: &Logger, + _log: &Logger, region_ids: Vec, ) -> DeleteResult { if region_ids.is_empty() { @@ -164,98 +162,79 @@ impl DataStore { #[error("Numeric error: {0}")] NumericError(String), } - type TxnError = TransactionError; - - // Retry this transaction until it succeeds. It's a little heavy in that - // there's a for loop inside that iterates over the datasets the - // argument regions belong to, and it often encounters the "retry - // transaction" error. - let transaction = { - |region_ids: Vec| async { - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - use db::schema::dataset::dsl as dataset_dsl; - use db::schema::region::dsl as region_dsl; - - // Remove the regions, collecting datasets they're from. - let datasets = diesel::delete(region_dsl::region) - .filter(region_dsl::id.eq_any(region_ids)) - .returning(region_dsl::dataset_id) - .get_results_async::(&conn).await?; - - // Update datasets to which the regions belonged. - for dataset in datasets { - let dataset_total_occupied_size: Option< - diesel::pg::data_types::PgNumeric, - > = region_dsl::region - .filter(region_dsl::dataset_id.eq(dataset)) - .select(diesel::dsl::sum( - region_dsl::block_size - * region_dsl::blocks_per_extent - * region_dsl::extent_count, - )) - .nullable() - .get_result_async(&conn).await?; - - let dataset_total_occupied_size: i64 = if let Some( - dataset_total_occupied_size, - ) = - dataset_total_occupied_size - { - let dataset_total_occupied_size: db::model::ByteCount = - dataset_total_occupied_size.try_into().map_err( - |e: anyhow::Error| { - TxnError::CustomError( - RegionDeleteError::NumericError( - e.to_string(), - ), - ) - }, - )?; - - dataset_total_occupied_size.into() - } else { - 0 - }; - - diesel::update(dataset_dsl::dataset) - .filter(dataset_dsl::id.eq(dataset)) - .set( - dataset_dsl::size_used - .eq(dataset_total_occupied_size), - ) - .execute_async(&conn).await?; - } - - Ok(()) - }) - .await - .map_err(|e: TxnError| { - if e.retry_transaction() { - BackoffError::transient(Error::internal_error( - &format!("Retryable transaction error {:?}", e) + let err = OptionalError::new(); + let conn = self.pool_connection_unauthorized().await?; + self.transaction_retry_wrapper("regions_hard_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + let region_ids = region_ids.clone(); + async move { + use db::schema::dataset::dsl as dataset_dsl; + use db::schema::region::dsl as region_dsl; + + // Remove the regions, collecting datasets they're from. + let datasets = diesel::delete(region_dsl::region) + .filter(region_dsl::id.eq_any(region_ids)) + .returning(region_dsl::dataset_id) + .get_results_async::(&conn).await?; + + // Update datasets to which the regions belonged. + for dataset in datasets { + let dataset_total_occupied_size: Option< + diesel::pg::data_types::PgNumeric, + > = region_dsl::region + .filter(region_dsl::dataset_id.eq(dataset)) + .select(diesel::dsl::sum( + region_dsl::block_size + * region_dsl::blocks_per_extent + * region_dsl::extent_count, )) + .nullable() + .get_result_async(&conn).await?; + + let dataset_total_occupied_size: i64 = if let Some( + dataset_total_occupied_size, + ) = + dataset_total_occupied_size + { + let dataset_total_occupied_size: db::model::ByteCount = + dataset_total_occupied_size.try_into().map_err( + |e: anyhow::Error| { + err.bail(RegionDeleteError::NumericError( + e.to_string(), + )) + }, + )?; + + dataset_total_occupied_size.into() } else { - BackoffError::Permanent(Error::internal_error( - &format!("Transaction error: {}", e) - )) + 0 + }; + + diesel::update(dataset_dsl::dataset) + .filter(dataset_dsl::id.eq(dataset)) + .set( + dataset_dsl::size_used + .eq(dataset_total_occupied_size), + ) + .execute_async(&conn).await?; + } + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + RegionDeleteError::NumericError(err) => { + return Error::internal_error( + &format!("Transaction error: {}", err) + ); } - }) - } - }; - - backoff::retry_notify( - backoff::retry_policy_internal_service_aggressive(), - || async { - let region_ids = region_ids.clone(); - transaction(region_ids).await - }, - |e: Error, delay| { - info!(log, "{:?}, trying again in {:?}", e, delay,); - }, - ) - .await + } + } + public_error_from_diesel(e, ErrorHandler::Server) + }) } /// Return the total occupied size for a dataset diff --git a/nexus/db-queries/src/db/datastore/service.rs b/nexus/db-queries/src/db/datastore/service.rs index 40bf250abe..df7ed27a6d 100644 --- a/nexus/db-queries/src/db/datastore/service.rs +++ b/nexus/db-queries/src/db/datastore/service.rs @@ -11,7 +11,9 @@ use crate::db; use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; +use crate::db::error::TransactionError; use crate::db::identity::Asset; use crate::db::model::Service; use crate::db::model::Sled; @@ -38,7 +40,12 @@ impl DataStore { service: Service, ) -> CreateResult { let conn = self.pool_connection_authorized(opctx).await?; - self.service_upsert_conn(&conn, service).await + self.service_upsert_conn(&conn, service).await.map_err(|e| match e { + TransactionError::CustomError(err) => err, + TransactionError::Database(err) => { + public_error_from_diesel(err, ErrorHandler::Server) + } + }) } /// Stores a new service in the database (using an existing db connection). @@ -46,7 +53,7 @@ impl DataStore { &self, conn: &async_bb8_diesel::Connection, service: Service, - ) -> CreateResult { + ) -> Result> { use db::schema::service::dsl; let service_id = service.id(); @@ -68,17 +75,24 @@ impl DataStore { .insert_and_get_result_async(conn) .await .map_err(|e| match e { - AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { - type_name: ResourceType::Sled, - lookup_type: LookupType::ById(sled_id), - }, - AsyncInsertError::DatabaseError(e) => public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Service, - &service_id.to_string(), - ), - ), + AsyncInsertError::CollectionNotFound => { + TransactionError::CustomError(Error::ObjectNotFound { + type_name: ResourceType::Sled, + lookup_type: LookupType::ById(sled_id), + }) + } + AsyncInsertError::DatabaseError(e) => { + if retryable(&e) { + return TransactionError::Database(e); + } + TransactionError::CustomError(public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Service, + &service_id.to_string(), + ), + )) + } }) } diff --git a/nexus/db-queries/src/db/datastore/silo.rs b/nexus/db-queries/src/db/datastore/silo.rs index ec3658c067..ab48ec458f 100644 --- a/nexus/db-queries/src/db/datastore/silo.rs +++ b/nexus/db-queries/src/db/datastore/silo.rs @@ -11,6 +11,7 @@ use crate::context::OpContext; use crate::db; use crate::db::datastore::RunnableQuery; use crate::db::error::public_error_from_diesel; +use crate::db::error::retryable; use crate::db::error::ErrorHandler; use crate::db::error::TransactionError; use crate::db::fixed_data::silo::{DEFAULT_SILO, INTERNAL_SILO}; @@ -123,15 +124,17 @@ impl DataStore { dns_update: DnsVersionUpdateBuilder, ) -> CreateResult { let conn = self.pool_connection_authorized(opctx).await?; - self.silo_create_conn( - &conn, - opctx, - nexus_opctx, - new_silo_params, - new_silo_dns_names, - dns_update, - ) - .await + let silo = self + .silo_create_conn( + &conn, + opctx, + nexus_opctx, + new_silo_params, + new_silo_dns_names, + dns_update, + ) + .await?; + Ok(silo) } pub async fn silo_create_conn( @@ -142,7 +145,7 @@ impl DataStore { new_silo_params: params::SiloCreate, new_silo_dns_names: &[String], dns_update: DnsVersionUpdateBuilder, - ) -> CreateResult { + ) -> Result> { let silo_id = Uuid::new_v4(); let silo_group_id = Uuid::new_v4(); @@ -199,71 +202,71 @@ impl DataStore { None }; - conn.transaction_async(|conn| async move { - let silo = silo_create_query - .get_result_async(&conn) - .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Silo, - new_silo_params.identity.name.as_str(), - ), - ) - })?; - self.virtual_provisioning_collection_create_on_connection( - &conn, - VirtualProvisioningCollection::new( - silo.id(), - CollectionTypeProvisioned::Silo, - ), - ) - .await?; - - if let Some(query) = silo_admin_group_ensure_query { - query.get_result_async(&conn).await?; - } - - if let Some(queries) = silo_admin_group_role_assignment_queries { - let (delete_old_query, insert_new_query) = queries; - delete_old_query.execute_async(&conn).await?; - insert_new_query.execute_async(&conn).await?; - } - - let certificates = new_silo_params - .tls_certificates - .into_iter() - .map(|c| { - Certificate::new( + let silo = conn + .transaction_async(|conn| async move { + let silo = silo_create_query + .get_result_async(&conn) + .await + .map_err(|e| { + if retryable(&e) { + return TransactionError::Database(e); + } + TransactionError::CustomError(public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Silo, + new_silo_params.identity.name.as_str(), + ), + )) + })?; + self.virtual_provisioning_collection_create_on_connection( + &conn, + VirtualProvisioningCollection::new( silo.id(), - Uuid::new_v4(), - ServiceKind::Nexus, - c, - new_silo_dns_names, - ) - }) - .collect::, _>>() - .map_err(Error::from)?; - { - use db::schema::certificate::dsl; - diesel::insert_into(dsl::certificate) - .values(certificates) - .execute_async(&conn) - .await?; - } - - self.dns_update(nexus_opctx, &conn, dns_update).await?; + CollectionTypeProvisioned::Silo, + ), + ) + .await?; - Ok(silo) - }) - .await - .map_err(|e| match e { - TransactionError::CustomError(e) => e, - TransactionError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + if let Some(query) = silo_admin_group_ensure_query { + query.get_result_async(&conn).await?; + } + + if let Some(queries) = silo_admin_group_role_assignment_queries + { + let (delete_old_query, insert_new_query) = queries; + delete_old_query.execute_async(&conn).await?; + insert_new_query.execute_async(&conn).await?; + } + + let certificates = new_silo_params + .tls_certificates + .into_iter() + .map(|c| { + Certificate::new( + silo.id(), + Uuid::new_v4(), + ServiceKind::Nexus, + c, + new_silo_dns_names, + ) + }) + .collect::, _>>() + .map_err(Error::from)?; + { + use db::schema::certificate::dsl; + diesel::insert_into(dsl::certificate) + .values(certificates) + .execute_async(&conn) + .await?; + } + + self.dns_update(nexus_opctx, &conn, dns_update).await?; + + Ok::>(silo) + }) + .await?; + Ok(silo) } pub async fn silos_list_by_id( @@ -380,7 +383,7 @@ impl DataStore { } self.virtual_provisioning_collection_delete_on_connection( - &conn, id, + &opctx.log, &conn, id, ) .await?; diff --git a/nexus/db-queries/src/db/datastore/silo_group.rs b/nexus/db-queries/src/db/datastore/silo_group.rs index 46f4aae7c9..29fcb7490b 100644 --- a/nexus/db-queries/src/db/datastore/silo_group.rs +++ b/nexus/db-queries/src/db/datastore/silo_group.rs @@ -145,35 +145,39 @@ impl DataStore { ) -> UpdateResult<()> { opctx.authorize(authz::Action::Modify, authz_silo_user).await?; - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - use db::schema::silo_group_membership::dsl; + let conn = self.pool_connection_authorized(opctx).await?; - // Delete existing memberships for user - let silo_user_id = authz_silo_user.id(); - diesel::delete(dsl::silo_group_membership) - .filter(dsl::silo_user_id.eq(silo_user_id)) - .execute_async(&conn) - .await?; + self.transaction_retry_wrapper("silo_group_membership_replace_for_user") + .transaction(&conn, |conn| { + let silo_group_ids = silo_group_ids.clone(); + async move { + use db::schema::silo_group_membership::dsl; + + // Delete existing memberships for user + let silo_user_id = authz_silo_user.id(); + diesel::delete(dsl::silo_group_membership) + .filter(dsl::silo_user_id.eq(silo_user_id)) + .execute_async(&conn) + .await?; - // Create new memberships for user - let silo_group_memberships: Vec< - db::model::SiloGroupMembership, - > = silo_group_ids - .iter() - .map(|group_id| db::model::SiloGroupMembership { - silo_group_id: *group_id, - silo_user_id, - }) - .collect(); + // Create new memberships for user + let silo_group_memberships: Vec< + db::model::SiloGroupMembership, + > = silo_group_ids + .iter() + .map(|group_id| db::model::SiloGroupMembership { + silo_group_id: *group_id, + silo_user_id, + }) + .collect(); - diesel::insert_into(dsl::silo_group_membership) - .values(silo_group_memberships) - .execute_async(&conn) - .await?; + diesel::insert_into(dsl::silo_group_membership) + .values(silo_group_memberships) + .execute_async(&conn) + .await?; - Ok(()) + Ok(()) + } }) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) diff --git a/nexus/db-queries/src/db/datastore/sled.rs b/nexus/db-queries/src/db/datastore/sled.rs index 406119a636..023384a9bf 100644 --- a/nexus/db-queries/src/db/datastore/sled.rs +++ b/nexus/db-queries/src/db/datastore/sled.rs @@ -10,13 +10,12 @@ use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::Sled; use crate::db::model::SledResource; use crate::db::model::SledUpdate; use crate::db::pagination::paginated; use crate::db::update_and_check::UpdateAndCheck; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; @@ -90,123 +89,141 @@ impl DataStore { enum SledReservationError { NotFound, } - type TxnError = TransactionError; - - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - use db::schema::sled_resource::dsl as resource_dsl; - // Check if resource ID already exists - if so, return it. - let old_resource = resource_dsl::sled_resource - .filter(resource_dsl::id.eq(resource_id)) - .select(SledResource::as_select()) - .limit(1) - .load_async(&conn) - .await?; - - if !old_resource.is_empty() { - return Ok(old_resource[0].clone()); - } - - // If it doesn't already exist, find a sled with enough space - // for the resources we're requesting. - use db::schema::sled::dsl as sled_dsl; - // This answers the boolean question: - // "Does the SUM of all hardware thread usage, plus the one we're trying - // to allocate, consume less threads than exists on the sled?" - let sled_has_space_for_threads = - (diesel::dsl::sql::(&format!( - "COALESCE(SUM(CAST({} as INT8)), 0)", - resource_dsl::hardware_threads::NAME - )) + resources.hardware_threads) - .le(sled_dsl::usable_hardware_threads); - - // This answers the boolean question: - // "Does the SUM of all RAM usage, plus the one we're trying - // to allocate, consume less RAM than exists on the sled?" - let sled_has_space_for_rss = - (diesel::dsl::sql::(&format!( - "COALESCE(SUM(CAST({} as INT8)), 0)", - resource_dsl::rss_ram::NAME - )) + resources.rss_ram) - .le(sled_dsl::usable_physical_ram); - - // Determine whether adding this service's reservoir allocation - // to what's allocated on the sled would avoid going over quota. - let sled_has_space_in_reservoir = - (diesel::dsl::sql::(&format!( - "COALESCE(SUM(CAST({} as INT8)), 0)", - resource_dsl::reservoir_ram::NAME - )) + resources.reservoir_ram) - .le(sled_dsl::reservoir_size); - - // Generate a query describing all of the sleds that have space - // for this reservation. - let mut sled_targets = sled_dsl::sled - .left_join( - resource_dsl::sled_resource - .on(resource_dsl::sled_id.eq(sled_dsl::id)), - ) - .group_by(sled_dsl::id) - .having( - sled_has_space_for_threads - .and(sled_has_space_for_rss) - .and(sled_has_space_in_reservoir), - ) - .filter(sled_dsl::time_deleted.is_null()) - // Filter out sleds that are not provisionable. - .filter( - sled_dsl::provision_state - .eq(db::model::SledProvisionState::Provisionable), - ) - .select(sled_dsl::id) - .into_boxed(); - - // Further constrain the sled IDs according to any caller- - // supplied constraints. - if let Some(must_select_from) = constraints.must_select_from() { - sled_targets = sled_targets - .filter(sled_dsl::id.eq_any(must_select_from.to_vec())); - } - sql_function!(fn random() -> diesel::sql_types::Float); - let sled_targets = sled_targets - .order(random()) - .limit(1) - .get_results_async::(&conn) - .await?; - - if sled_targets.is_empty() { - return Err(TxnError::CustomError( - SledReservationError::NotFound, - )); + let err = OptionalError::new(); + + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("sled_reservation_create") + .transaction(&conn, |conn| { + // Clone variables into retryable function + let err = err.clone(); + let constraints = constraints.clone(); + let resources = resources.clone(); + + async move { + use db::schema::sled_resource::dsl as resource_dsl; + // Check if resource ID already exists - if so, return it. + let old_resource = resource_dsl::sled_resource + .filter(resource_dsl::id.eq(resource_id)) + .select(SledResource::as_select()) + .limit(1) + .load_async(&conn) + .await?; + + if !old_resource.is_empty() { + return Ok(old_resource[0].clone()); + } + + // If it doesn't already exist, find a sled with enough space + // for the resources we're requesting. + use db::schema::sled::dsl as sled_dsl; + // This answers the boolean question: + // "Does the SUM of all hardware thread usage, plus the one we're trying + // to allocate, consume less threads than exists on the sled?" + let sled_has_space_for_threads = + (diesel::dsl::sql::( + &format!( + "COALESCE(SUM(CAST({} as INT8)), 0)", + resource_dsl::hardware_threads::NAME + ), + ) + resources.hardware_threads) + .le(sled_dsl::usable_hardware_threads); + + // This answers the boolean question: + // "Does the SUM of all RAM usage, plus the one we're trying + // to allocate, consume less RAM than exists on the sled?" + let sled_has_space_for_rss = + (diesel::dsl::sql::( + &format!( + "COALESCE(SUM(CAST({} as INT8)), 0)", + resource_dsl::rss_ram::NAME + ), + ) + resources.rss_ram) + .le(sled_dsl::usable_physical_ram); + + // Determine whether adding this service's reservoir allocation + // to what's allocated on the sled would avoid going over quota. + let sled_has_space_in_reservoir = + (diesel::dsl::sql::( + &format!( + "COALESCE(SUM(CAST({} as INT8)), 0)", + resource_dsl::reservoir_ram::NAME + ), + ) + resources.reservoir_ram) + .le(sled_dsl::reservoir_size); + + // Generate a query describing all of the sleds that have space + // for this reservation. + let mut sled_targets = + sled_dsl::sled + .left_join( + resource_dsl::sled_resource + .on(resource_dsl::sled_id.eq(sled_dsl::id)), + ) + .group_by(sled_dsl::id) + .having( + sled_has_space_for_threads + .and(sled_has_space_for_rss) + .and(sled_has_space_in_reservoir), + ) + .filter(sled_dsl::time_deleted.is_null()) + // Filter out sleds that are not provisionable. + .filter(sled_dsl::provision_state.eq( + db::model::SledProvisionState::Provisionable, + )) + .select(sled_dsl::id) + .into_boxed(); + + // Further constrain the sled IDs according to any caller- + // supplied constraints. + if let Some(must_select_from) = + constraints.must_select_from() + { + sled_targets = sled_targets.filter( + sled_dsl::id.eq_any(must_select_from.to_vec()), + ); + } + + sql_function!(fn random() -> diesel::sql_types::Float); + let sled_targets = sled_targets + .order(random()) + .limit(1) + .get_results_async::(&conn) + .await?; + + if sled_targets.is_empty() { + return Err(err.bail(SledReservationError::NotFound)); + } + + // Create a SledResource record, associate it with the target + // sled. + let resource = SledResource::new( + resource_id, + sled_targets[0], + resource_kind, + resources, + ); + + diesel::insert_into(resource_dsl::sled_resource) + .values(resource) + .returning(SledResource::as_returning()) + .get_result_async(&conn) + .await } - - // Create a SledResource record, associate it with the target - // sled. - let resource = SledResource::new( - resource_id, - sled_targets[0], - resource_kind, - resources, - ); - - Ok(diesel::insert_into(resource_dsl::sled_resource) - .values(resource) - .returning(SledResource::as_returning()) - .get_result_async(&conn) - .await?) }) .await - .map_err(|e| match e { - TxnError::CustomError(SledReservationError::NotFound) => { - external::Error::unavail( - "No sleds can fit the requested instance", - ) - } - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SledReservationError::NotFound => { + return external::Error::unavail( + "No sleds can fit the requested instance", + ); + } + } } + public_error_from_diesel(e, ErrorHandler::Server) }) } diff --git a/nexus/db-queries/src/db/datastore/snapshot.rs b/nexus/db-queries/src/db/datastore/snapshot.rs index 7c03e4bd40..7a9eb8d2bc 100644 --- a/nexus/db-queries/src/db/datastore/snapshot.rs +++ b/nexus/db-queries/src/db/datastore/snapshot.rs @@ -20,8 +20,7 @@ use crate::db::model::SnapshotState; use crate::db::pagination::paginated; use crate::db::update_and_check::UpdateAndCheck; use crate::db::update_and_check::UpdateStatus; -use crate::db::TransactionError; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; @@ -48,114 +47,99 @@ impl DataStore { let gen = snapshot.gen; opctx.authorize(authz::Action::CreateChild, authz_project).await?; - #[derive(Debug, thiserror::Error)] - pub enum CustomError { - #[error("Resource already exists")] - ResourceAlreadyExists, - - #[error("saw AsyncInsertError")] - InsertError(AsyncInsertError), - } - - type TxnError = TransactionError; - - let snapshot_name = snapshot.name().to_string(); let project_id = snapshot.project_id; - let snapshot: Snapshot = self - .pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - use db::schema::snapshot::dsl; + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; - // If an undeleted snapshot exists in the database with the - // same name and project but a different id to the snapshot - // this function was passed as an argument, then return an - // error here. - // - // As written below, - // - // .on_conflict((dsl::project_id, dsl::name)) - // .filter_target(dsl::time_deleted.is_null()) - // .do_update() - // .set(dsl::time_modified.eq(dsl::time_modified)) - // - // will set any existing record's `time_modified` if the - // project id and name match, even if the snapshot ID does - // not match. diesel supports adding a filter below like so - // (marked with >>): - // - // .on_conflict((dsl::project_id, dsl::name)) - // .filter_target(dsl::time_deleted.is_null()) - // .do_update() - // .set(dsl::time_modified.eq(dsl::time_modified)) - // >> .filter(dsl::id.eq(snapshot.id())) - // - // which will restrict the `insert_into`'s set so that it - // only applies if the snapshot ID matches. But, - // AsyncInsertError does not have a ObjectAlreadyExists - // variant, so this will be returned as CollectionNotFound - // due to the `insert_into` failing. - // - // If this function is passed a snapshot with an ID that - // does not match, but a project and name that does, return - // ObjectAlreadyExists here. + let snapshot: Snapshot = self + .transaction_retry_wrapper("project_ensure_snapshot") + .transaction(&conn, |conn| { + let err = err.clone(); + let snapshot = snapshot.clone(); + let snapshot_name = snapshot.name().to_string(); + async move { + use db::schema::snapshot::dsl; - let existing_snapshot_id: Option = dsl::snapshot - .filter(dsl::time_deleted.is_null()) - .filter(dsl::name.eq(snapshot.name().to_string())) - .filter(dsl::project_id.eq(snapshot.project_id)) - .select(dsl::id) - .limit(1) - .first_async(&conn) - .await - .optional()?; + // If an undeleted snapshot exists in the database with the + // same name and project but a different id to the snapshot + // this function was passed as an argument, then return an + // error here. + // + // As written below, + // + // .on_conflict((dsl::project_id, dsl::name)) + // .filter_target(dsl::time_deleted.is_null()) + // .do_update() + // .set(dsl::time_modified.eq(dsl::time_modified)) + // + // will set any existing record's `time_modified` if the + // project id and name match, even if the snapshot ID does + // not match. diesel supports adding a filter below like so + // (marked with >>): + // + // .on_conflict((dsl::project_id, dsl::name)) + // .filter_target(dsl::time_deleted.is_null()) + // .do_update() + // .set(dsl::time_modified.eq(dsl::time_modified)) + // >> .filter(dsl::id.eq(snapshot.id())) + // + // which will restrict the `insert_into`'s set so that it + // only applies if the snapshot ID matches. But, + // AsyncInsertError does not have a ObjectAlreadyExists + // variant, so this will be returned as CollectionNotFound + // due to the `insert_into` failing. + // + // If this function is passed a snapshot with an ID that + // does not match, but a project and name that does, return + // ObjectAlreadyExists here. - if let Some(existing_snapshot_id) = existing_snapshot_id { - if existing_snapshot_id != snapshot.id() { - return Err(TransactionError::CustomError( - CustomError::ResourceAlreadyExists, - )); - } - } + let existing_snapshot_id: Option = dsl::snapshot + .filter(dsl::time_deleted.is_null()) + .filter(dsl::name.eq(snapshot.name().to_string())) + .filter(dsl::project_id.eq(snapshot.project_id)) + .select(dsl::id) + .limit(1) + .first_async(&conn) + .await + .optional()?; - Project::insert_resource( - project_id, - diesel::insert_into(dsl::snapshot) - .values(snapshot) - .on_conflict((dsl::project_id, dsl::name)) - .filter_target(dsl::time_deleted.is_null()) - .do_update() - .set(dsl::time_modified.eq(dsl::time_modified)), - ) - .insert_and_get_result_async(&conn) - .await - .map_err(|e| { - TransactionError::CustomError(CustomError::InsertError(e)) - }) - }) - .await - .map_err(|e: TxnError| match e { - TxnError::CustomError(e) => match e { - CustomError::ResourceAlreadyExists => { - Error::ObjectAlreadyExists { - type_name: ResourceType::Snapshot, - object_name: snapshot_name, + if let Some(existing_snapshot_id) = existing_snapshot_id { + if existing_snapshot_id != snapshot.id() { + return Err(err.bail(Error::ObjectAlreadyExists { + type_name: ResourceType::Snapshot, + object_name: snapshot_name, + })); } } - CustomError::InsertError(e) => match e { + + Project::insert_resource( + project_id, + diesel::insert_into(dsl::snapshot) + .values(snapshot) + .on_conflict((dsl::project_id, dsl::name)) + .filter_target(dsl::time_deleted.is_null()) + .do_update() + .set(dsl::time_modified.eq(dsl::time_modified)), + ) + .insert_and_get_result_async(&conn) + .await + .map_err(|e| match e { AsyncInsertError::CollectionNotFound => { - Error::ObjectNotFound { + err.bail(Error::ObjectNotFound { type_name: ResourceType::Project, lookup_type: LookupType::ById(project_id), - } - } - AsyncInsertError::DatabaseError(e) => { - public_error_from_diesel(e, ErrorHandler::Server) + }) } - }, - }, - TxnError::Database(e) => { + AsyncInsertError::DatabaseError(e) => e, + }) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + err + } else { public_error_from_diesel(e, ErrorHandler::Server) } })?; diff --git a/nexus/db-queries/src/db/datastore/switch_interface.rs b/nexus/db-queries/src/db/datastore/switch_interface.rs index 88cff50471..67f16fa08f 100644 --- a/nexus/db-queries/src/db/datastore/switch_interface.rs +++ b/nexus/db-queries/src/db/datastore/switch_interface.rs @@ -11,11 +11,10 @@ use crate::db::datastore::address_lot::{ }; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::LoopbackAddress; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; -use diesel::result::Error as DieselError; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use ipnetwork::IpNetwork; use nexus_types::external_api::params::LoopbackAddressCreate; @@ -40,80 +39,78 @@ impl DataStore { ReserveBlock(ReserveBlockError), } - type TxnError = TransactionError; - let conn = self.pool_connection_authorized(opctx).await?; let inet = IpNetwork::new(params.address, params.mask) .map_err(|_| Error::invalid_request("invalid address"))?; + let err = OptionalError::new(); + // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - let lot_id = authz_address_lot.id(); - let (block, rsvd_block) = - crate::db::datastore::address_lot::try_reserve_block( - lot_id, - inet.ip().into(), - params.anycast, - &conn, - ) - .await - .map_err(|e| match e { - ReserveBlockTxnError::CustomError(err) => { - TxnError::CustomError( - LoopbackAddressCreateError::ReserveBlock(err), + self.transaction_retry_wrapper("loopback_address_create") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + let lot_id = authz_address_lot.id(); + let (block, rsvd_block) = + crate::db::datastore::address_lot::try_reserve_block( + lot_id, + inet.ip().into(), + params.anycast, + &conn, ) + .await + .map_err(|e| match e { + ReserveBlockTxnError::CustomError(e) => err.bail( + LoopbackAddressCreateError::ReserveBlock(e), + ), + ReserveBlockTxnError::Database(e) => e, + })?; + + // Address block reserved, now create the loopback address. + + let addr = LoopbackAddress::new( + id, + block.id, + rsvd_block.id, + params.rack_id, + params.switch_location.to_string(), + inet, + params.anycast, + ); + + let db_addr: LoopbackAddress = + diesel::insert_into(dsl::loopback_address) + .values(addr) + .returning(LoopbackAddress::as_returning()) + .get_result_async(&conn) + .await?; + + Ok(db_addr) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + LoopbackAddressCreateError::ReserveBlock( + ReserveBlockError::AddressUnavailable, + ) => Error::invalid_request("address unavailable"), + LoopbackAddressCreateError::ReserveBlock( + ReserveBlockError::AddressNotInLot, + ) => Error::invalid_request("address not in lot"), } - ReserveBlockTxnError::Database(err) => { - TxnError::Database(err) - } - })?; - - // Address block reserved, now create the loopback address. - - let addr = LoopbackAddress::new( - id, - block.id, - rsvd_block.id, - params.rack_id, - params.switch_location.to_string(), - inet, - params.anycast, - ); - - let db_addr: LoopbackAddress = - diesel::insert_into(dsl::loopback_address) - .values(addr) - .returning(LoopbackAddress::as_returning()) - .get_result_async(&conn) - .await?; - - Ok(db_addr) - }) - .await - .map_err(|e| match e { - TxnError::CustomError( - LoopbackAddressCreateError::ReserveBlock( - ReserveBlockError::AddressUnavailable, - ), - ) => Error::invalid_request("address unavailable"), - TxnError::CustomError( - LoopbackAddressCreateError::ReserveBlock( - ReserveBlockError::AddressNotInLot, - ), - ) => Error::invalid_request("address not in lot"), - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::LoopbackAddress, - &format!("lo {}", inet), - ), - ), - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, - }) + } else { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::LoopbackAddress, + &format!("lo {}", inet), + ), + ) + } + }) } pub async fn loopback_address_delete( @@ -130,22 +127,23 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - let la = diesel::delete(dsl::loopback_address) - .filter(dsl::id.eq(id)) - .returning(LoopbackAddress::as_returning()) - .get_result_async(&conn) - .await?; - - diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) - .filter(rsvd_block_dsl::id.eq(la.rsvd_address_lot_block_id)) - .execute_async(&conn) - .await?; - - Ok(()) - }) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + self.transaction_retry_wrapper("loopback_address_delete") + .transaction(&conn, |conn| async move { + let la = diesel::delete(dsl::loopback_address) + .filter(dsl::id.eq(id)) + .returning(LoopbackAddress::as_returning()) + .get_result_async(&conn) + .await?; + + diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) + .filter(rsvd_block_dsl::id.eq(la.rsvd_address_lot_block_id)) + .execute_async(&conn) + .await?; + + Ok(()) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub async fn loopback_address_get( diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index 6bd4e61f70..221feee23c 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -11,7 +11,6 @@ use crate::db::datastore::address_lot::{ use crate::db::datastore::UpdatePrecondition; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::model::{ LldpServiceConfig, Name, SwitchInterfaceConfig, SwitchPort, SwitchPortAddressConfig, SwitchPortBgpPeerConfig, SwitchPortConfig, @@ -20,8 +19,8 @@ use crate::db::model::{ SwitchVlanInterfaceConfig, }; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; -use diesel::result::Error as DieselError; +use crate::transaction_retry::OptionalError; +use async_bb8_diesel::AsyncRunQueryDsl; use diesel::{ CombineDsl, ExpressionMethods, JoinOnDsl, NullableExpressionMethods, QueryDsl, SelectableHelper, @@ -163,283 +162,285 @@ impl DataStore { BgpConfigNotFound, ReserveBlock(ReserveBlockError), } - type TxnError = TransactionError; type SpsCreateError = SwitchPortSettingsCreateError; + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - // create the top level port settings object - let port_settings = match id { - Some(id) => SwitchPortSettings::with_id(id, ¶ms.identity), - None => SwitchPortSettings::new(¶ms.identity), - }; - //let port_settings = SwitchPortSettings::new(¶ms.identity); - let db_port_settings: SwitchPortSettings = - diesel::insert_into(port_settings_dsl::switch_port_settings) - .values(port_settings) - .returning(SwitchPortSettings::as_returning()) - .get_result_async(&conn) - .await?; - - let psid = db_port_settings.identity.id; - - // add the port config - let port_config = SwitchPortConfig::new( - psid, - params.port_config.geometry.into(), - ); - - let db_port_config: SwitchPortConfig = - diesel::insert_into(port_config_dsl::switch_port_settings_port_config) - .values(port_config) - .returning(SwitchPortConfig::as_returning()) - .get_result_async(&conn) - .await?; - - let mut result = SwitchPortSettingsCombinedResult{ - settings: db_port_settings, - groups: Vec::new(), - port: db_port_config, - links: Vec::new(), - link_lldp: Vec::new(), - interfaces: Vec::new(), - vlan_interfaces: Vec::new(), - routes: Vec::new(), - bgp_peers: Vec::new(), - addresses: Vec::new(), - }; - - //TODO validate link configs consistent with port geometry. - // - https://github.com/oxidecomputer/omicron/issues/2816 - - let mut lldp_config = Vec::with_capacity(params.links.len()); - let mut link_config = Vec::with_capacity(params.links.len()); - - for (link_name, c) in ¶ms.links { - let lldp_config_id = match c.lldp.lldp_config { - Some(_) => todo!(), // TODO actual lldp support - None => None, - }; - let lldp_svc_config = - LldpServiceConfig::new(c.lldp.enabled, lldp_config_id); - - lldp_config.push(lldp_svc_config.clone()); - link_config.push(SwitchPortLinkConfig::new( - psid, - lldp_svc_config.id, - link_name.clone(), - c.mtu, - c.fec.into(), - c.speed.into(), - c.autoneg, - )); - } - result.link_lldp = - diesel::insert_into(lldp_config_dsl::lldp_service_config) - .values(lldp_config.clone()) - .returning(LldpServiceConfig::as_returning()) - .get_results_async(&conn) - .await?; - result.links = - diesel::insert_into( - link_config_dsl::switch_port_settings_link_config) - .values(link_config) - .returning(SwitchPortLinkConfig::as_returning()) - .get_results_async(&conn) - .await?; - - let mut interface_config = Vec::with_capacity(params.interfaces.len()); - let mut vlan_interface_config = Vec::new(); - for (interface_name, i) in ¶ms.interfaces { - let ifx_config = SwitchInterfaceConfig::new( - psid, - interface_name.clone(), - i.v6_enabled, - i.kind.into(), - ); - interface_config.push(ifx_config.clone()); - if let params::SwitchInterfaceKind::Vlan(vlan_if) = i.kind { - vlan_interface_config.push(SwitchVlanInterfaceConfig::new( - ifx_config.id, - vlan_if.vid, - )); - } - } - result.interfaces = - diesel::insert_into( - interface_config_dsl::switch_port_settings_interface_config) - .values(interface_config) - .returning(SwitchInterfaceConfig::as_returning()) - .get_results_async(&conn) - .await?; - result.vlan_interfaces = - diesel::insert_into(vlan_config_dsl::switch_vlan_interface_config) - .values(vlan_interface_config) - .returning(SwitchVlanInterfaceConfig::as_returning()) - .get_results_async(&conn) - .await?; - - - let mut route_config = Vec::with_capacity(params.routes.len()); - - for (interface_name, r) in ¶ms.routes { - for route in &r.routes { - route_config.push(SwitchPortRouteConfig::new( - psid, - interface_name.clone(), - route.dst.into(), - route.gw.into(), - route.vid.map(Into::into), - )); - } - } - result.routes = - diesel::insert_into( - route_config_dsl::switch_port_settings_route_config) - .values(route_config) - .returning(SwitchPortRouteConfig::as_returning()) - .get_results_async(&conn) - .await?; - - let mut bgp_peer_config = Vec::new(); - for (interface_name, peer_config) in ¶ms.bgp_peers { - for p in &peer_config.peers { - use db::schema::bgp_config; - let bgp_config_id = match &p.bgp_config { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - bgp_config_dsl::bgp_config - .filter(bgp_config::time_deleted.is_null()) - .filter(bgp_config::name.eq(name)) - .select(bgp_config::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpConfigNotFound, - ) - )? - } + self.transaction_retry_wrapper("switch_port_settings_create") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + // create the top level port settings object + let port_settings = match id { + Some(id) => SwitchPortSettings::with_id(id, ¶ms.identity), + None => SwitchPortSettings::new(¶ms.identity), }; - - bgp_peer_config.push(SwitchPortBgpPeerConfig::new( + //let port_settings = SwitchPortSettings::new(¶ms.identity); + let db_port_settings: SwitchPortSettings = + diesel::insert_into(port_settings_dsl::switch_port_settings) + .values(port_settings) + .returning(SwitchPortSettings::as_returning()) + .get_result_async(&conn) + .await?; + + let psid = db_port_settings.identity.id; + + // add the port config + let port_config = SwitchPortConfig::new( psid, - bgp_config_id, - interface_name.clone(), - p.addr.into(), - p.hold_time.into(), - p.idle_hold_time.into(), - p.delay_open.into(), - p.connect_retry.into(), - p.keepalive.into(), - )); + params.port_config.geometry.into(), + ); + + let db_port_config: SwitchPortConfig = + diesel::insert_into(port_config_dsl::switch_port_settings_port_config) + .values(port_config) + .returning(SwitchPortConfig::as_returning()) + .get_result_async(&conn) + .await?; + + let mut result = SwitchPortSettingsCombinedResult{ + settings: db_port_settings, + groups: Vec::new(), + port: db_port_config, + links: Vec::new(), + link_lldp: Vec::new(), + interfaces: Vec::new(), + vlan_interfaces: Vec::new(), + routes: Vec::new(), + bgp_peers: Vec::new(), + addresses: Vec::new(), + }; - } - } - result.bgp_peers = - diesel::insert_into( - bgp_peer_dsl::switch_port_settings_bgp_peer_config) - .values(bgp_peer_config) - .returning(SwitchPortBgpPeerConfig::as_returning()) - .get_results_async(&conn) - .await?; + //TODO validate link configs consistent with port geometry. + // - https://github.com/oxidecomputer/omicron/issues/2816 + + let mut lldp_config = Vec::with_capacity(params.links.len()); + let mut link_config = Vec::with_capacity(params.links.len()); + + for (link_name, c) in ¶ms.links { + let lldp_config_id = match c.lldp.lldp_config { + Some(_) => todo!(), // TODO actual lldp support + None => None, + }; + let lldp_svc_config = + LldpServiceConfig::new(c.lldp.enabled, lldp_config_id); + + lldp_config.push(lldp_svc_config.clone()); + link_config.push(SwitchPortLinkConfig::new( + psid, + lldp_svc_config.id, + link_name.clone(), + c.mtu, + c.fec.into(), + c.speed.into(), + c.autoneg, + )); + } + result.link_lldp = + diesel::insert_into(lldp_config_dsl::lldp_service_config) + .values(lldp_config.clone()) + .returning(LldpServiceConfig::as_returning()) + .get_results_async(&conn) + .await?; + result.links = + diesel::insert_into( + link_config_dsl::switch_port_settings_link_config) + .values(link_config) + .returning(SwitchPortLinkConfig::as_returning()) + .get_results_async(&conn) + .await?; + + let mut interface_config = Vec::with_capacity(params.interfaces.len()); + let mut vlan_interface_config = Vec::new(); + for (interface_name, i) in ¶ms.interfaces { + let ifx_config = SwitchInterfaceConfig::new( + psid, + interface_name.clone(), + i.v6_enabled, + i.kind.into(), + ); + interface_config.push(ifx_config.clone()); + if let params::SwitchInterfaceKind::Vlan(vlan_if) = i.kind { + vlan_interface_config.push(SwitchVlanInterfaceConfig::new( + ifx_config.id, + vlan_if.vid, + )); + } + } + result.interfaces = + diesel::insert_into( + interface_config_dsl::switch_port_settings_interface_config) + .values(interface_config) + .returning(SwitchInterfaceConfig::as_returning()) + .get_results_async(&conn) + .await?; + result.vlan_interfaces = + diesel::insert_into(vlan_config_dsl::switch_vlan_interface_config) + .values(vlan_interface_config) + .returning(SwitchVlanInterfaceConfig::as_returning()) + .get_results_async(&conn) + .await?; + + let mut route_config = Vec::with_capacity(params.routes.len()); + + for (interface_name, r) in ¶ms.routes { + for route in &r.routes { + route_config.push(SwitchPortRouteConfig::new( + psid, + interface_name.clone(), + route.dst.into(), + route.gw.into(), + route.vid.map(Into::into), + )); + } + } + result.routes = + diesel::insert_into( + route_config_dsl::switch_port_settings_route_config) + .values(route_config) + .returning(SwitchPortRouteConfig::as_returning()) + .get_results_async(&conn) + .await?; + + let mut bgp_peer_config = Vec::new(); + for (interface_name, peer_config) in ¶ms.bgp_peers { + for p in &peer_config.peers { + use db::schema::bgp_config; + let bgp_config_id = match &p.bgp_config { + NameOrId::Id(id) => *id, + NameOrId::Name(name) => { + let name = name.to_string(); + bgp_config_dsl::bgp_config + .filter(bgp_config::time_deleted.is_null()) + .filter(bgp_config::name.eq(name)) + .select(bgp_config::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or( + diesel_error, + SwitchPortSettingsCreateError::BgpConfigNotFound + ) + })? + } + }; + + bgp_peer_config.push(SwitchPortBgpPeerConfig::new( + psid, + bgp_config_id, + interface_name.clone(), + p.addr.into(), + p.hold_time.into(), + p.idle_hold_time.into(), + p.delay_open.into(), + p.connect_retry.into(), + p.keepalive.into(), + )); - let mut address_config = Vec::new(); - use db::schema::address_lot; - for (interface_name, a) in ¶ms.addresses { - for address in &a.addresses { - let address_lot_id = match &address.address_lot { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - address_lot_dsl::address_lot - .filter(address_lot::time_deleted.is_null()) - .filter(address_lot::name.eq(name)) - .select(address_lot::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| - TxnError::CustomError( - SwitchPortSettingsCreateError::AddressLotNotFound, - ) - )? } - }; - // TODO: Reduce DB round trips needed for reserving ip blocks - // https://github.com/oxidecomputer/omicron/issues/3060 - let (block, rsvd_block) = - crate::db::datastore::address_lot::try_reserve_block( - address_lot_id, - address.address.ip().into(), - // TODO: Should we allow anycast addresses for switch_ports? - // anycast - false, - &conn - ) - .await - .map_err(|e| match e { - ReserveBlockTxnError::CustomError(err) => { - TxnError::CustomError( - SwitchPortSettingsCreateError::ReserveBlock(err) + } + result.bgp_peers = + diesel::insert_into( + bgp_peer_dsl::switch_port_settings_bgp_peer_config) + .values(bgp_peer_config) + .returning(SwitchPortBgpPeerConfig::as_returning()) + .get_results_async(&conn) + .await?; + + let mut address_config = Vec::new(); + use db::schema::address_lot; + for (interface_name, a) in ¶ms.addresses { + for address in &a.addresses { + let address_lot_id = match &address.address_lot { + NameOrId::Id(id) => *id, + NameOrId::Name(name) => { + let name = name.to_string(); + address_lot_dsl::address_lot + .filter(address_lot::time_deleted.is_null()) + .filter(address_lot::name.eq(name)) + .select(address_lot::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or( + diesel_error, + SwitchPortSettingsCreateError::AddressLotNotFound + ) + })? + } + }; + // TODO: Reduce DB round trips needed for reserving ip blocks + // https://github.com/oxidecomputer/omicron/issues/3060 + let (block, rsvd_block) = + crate::db::datastore::address_lot::try_reserve_block( + address_lot_id, + address.address.ip().into(), + // TODO: Should we allow anycast addresses for switch_ports? + // anycast + false, + &conn ) - } - ReserveBlockTxnError::Database(err) => TxnError::Database(err), - })?; - - address_config.push(SwitchPortAddressConfig::new( - psid, - block.id, - rsvd_block.id, - address.address.into(), - interface_name.clone(), - )); + .await + .map_err(|e| match e { + ReserveBlockTxnError::CustomError(e) => { + err.bail(SwitchPortSettingsCreateError::ReserveBlock(e)) + } + ReserveBlockTxnError::Database(e) => e, + })?; + + address_config.push(SwitchPortAddressConfig::new( + psid, + block.id, + rsvd_block.id, + address.address.into(), + interface_name.clone(), + )); + } + } + result.addresses = + diesel::insert_into( + address_config_dsl::switch_port_settings_address_config) + .values(address_config) + .returning(SwitchPortAddressConfig::as_returning()) + .get_results_async(&conn) + .await?; + + Ok(result) } } - result.addresses = - diesel::insert_into( - address_config_dsl::switch_port_settings_address_config) - .values(address_config) - .returning(SwitchPortAddressConfig::as_returning()) - .get_results_async(&conn) - .await?; - - Ok(result) - }) + ) .await - .map_err(|e| match e { - TxnError::CustomError(SpsCreateError::AddressLotNotFound) => { - Error::invalid_request("AddressLot not found") - } - TxnError::CustomError(SpsCreateError::BgpConfigNotFound) => { - Error::invalid_request("BGP config not found") - } - TxnError::CustomError( - SwitchPortSettingsCreateError::ReserveBlock( - ReserveBlockError::AddressUnavailable - ) - ) => Error::invalid_request("address unavailable"), - TxnError::CustomError( - SwitchPortSettingsCreateError::ReserveBlock( - ReserveBlockError::AddressNotInLot - ) - ) => Error::invalid_request("address not in lot"), - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => public_error_from_diesel( + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SpsCreateError::AddressLotNotFound => { + Error::invalid_request("AddressLot not found") + } + SpsCreateError::BgpConfigNotFound => { + Error::invalid_request("BGP config not found") + } + SwitchPortSettingsCreateError::ReserveBlock( + ReserveBlockError::AddressUnavailable + ) => Error::invalid_request("address unavailable"), + SwitchPortSettingsCreateError::ReserveBlock( + ReserveBlockError::AddressNotInLot + ) => Error::invalid_request("address not in lot"), + } + } else { + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::SwitchPortSettings, params.identity.name.as_str(), ), - ), - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, + ) + } }) } @@ -454,7 +455,6 @@ impl DataStore { enum SwitchPortSettingsDeleteError { SwitchPortSettingsNotFound, } - type TxnError = TransactionError; let conn = self.pool_connection_authorized(opctx).await?; @@ -463,173 +463,178 @@ impl DataStore { Some(name_or_id) => name_or_id, }; + let err = OptionalError::new(); + // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - - use db::schema::switch_port_settings; - let id = match selector { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - port_settings_dsl::switch_port_settings - .filter(switch_port_settings::time_deleted.is_null()) - .filter(switch_port_settings::name.eq(name)) - .select(switch_port_settings::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| - TxnError::CustomError( - SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound, - ) - )? - } - }; + self.transaction_retry_wrapper("switch_port_settings_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::switch_port_settings; + let id = match selector { + NameOrId::Id(id) => *id, + NameOrId::Name(name) => { + let name = name.to_string(); + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::name.eq(name)) + .select(switch_port_settings::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or( + diesel_error, + SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound + ) + })? + } + }; - // delete the top level port settings object - diesel::delete(port_settings_dsl::switch_port_settings) - .filter(switch_port_settings::id.eq(id)) - .execute_async(&conn) - .await?; + // delete the top level port settings object + diesel::delete(port_settings_dsl::switch_port_settings) + .filter(switch_port_settings::id.eq(id)) + .execute_async(&conn) + .await?; - // delete the port config object - use db::schema::switch_port_settings_port_config::{ - self as sps_port_config, dsl as port_config_dsl, - }; - diesel::delete(port_config_dsl::switch_port_settings_port_config) - .filter(sps_port_config::port_settings_id.eq(id)) - .execute_async(&conn) - .await?; + // delete the port config object + use db::schema::switch_port_settings_port_config::{ + self as sps_port_config, dsl as port_config_dsl, + }; + diesel::delete(port_config_dsl::switch_port_settings_port_config) + .filter(sps_port_config::port_settings_id.eq(id)) + .execute_async(&conn) + .await?; - // delete the link configs - use db::schema::switch_port_settings_link_config::{ - self as sps_link_config, dsl as link_config_dsl, - }; - let links: Vec = - diesel::delete( - link_config_dsl::switch_port_settings_link_config - ) - .filter( - sps_link_config::port_settings_id.eq(id) - ) - .returning(SwitchPortLinkConfig::as_returning()) - .get_results_async(&conn) - .await?; + // delete the link configs + use db::schema::switch_port_settings_link_config::{ + self as sps_link_config, dsl as link_config_dsl, + }; + let links: Vec = + diesel::delete( + link_config_dsl::switch_port_settings_link_config + ) + .filter( + sps_link_config::port_settings_id.eq(id) + ) + .returning(SwitchPortLinkConfig::as_returning()) + .get_results_async(&conn) + .await?; - // delete lldp configs - use db::schema::lldp_service_config::{self, dsl as lldp_config_dsl}; - let lldp_svc_ids: Vec = links - .iter() - .map(|link| link.lldp_service_config_id) - .collect(); - diesel::delete(lldp_config_dsl::lldp_service_config) - .filter(lldp_service_config::id.eq_any(lldp_svc_ids)) - .execute_async(&conn) - .await?; + // delete lldp configs + use db::schema::lldp_service_config::{self, dsl as lldp_config_dsl}; + let lldp_svc_ids: Vec = links + .iter() + .map(|link| link.lldp_service_config_id) + .collect(); + diesel::delete(lldp_config_dsl::lldp_service_config) + .filter(lldp_service_config::id.eq_any(lldp_svc_ids)) + .execute_async(&conn) + .await?; - // delete interface configs - use db::schema::switch_port_settings_interface_config::{ - self as sps_interface_config, dsl as interface_config_dsl, - }; + // delete interface configs + use db::schema::switch_port_settings_interface_config::{ + self as sps_interface_config, dsl as interface_config_dsl, + }; - let interfaces: Vec = - diesel::delete( - interface_config_dsl::switch_port_settings_interface_config - ) - .filter( - sps_interface_config::port_settings_id.eq(id) - ) - .returning(SwitchInterfaceConfig::as_returning()) - .get_results_async(&conn) - .await?; + let interfaces: Vec = + diesel::delete( + interface_config_dsl::switch_port_settings_interface_config + ) + .filter( + sps_interface_config::port_settings_id.eq(id) + ) + .returning(SwitchInterfaceConfig::as_returning()) + .get_results_async(&conn) + .await?; - // delete any vlan interfaces - use db::schema::switch_vlan_interface_config::{ - self, dsl as vlan_config_dsl, - }; - let interface_ids: Vec = interfaces - .iter() - .map(|interface| interface.id) - .collect(); - - diesel::delete(vlan_config_dsl::switch_vlan_interface_config) - .filter( - switch_vlan_interface_config::interface_config_id.eq_any( - interface_ids + // delete any vlan interfaces + use db::schema::switch_vlan_interface_config::{ + self, dsl as vlan_config_dsl, + }; + let interface_ids: Vec = interfaces + .iter() + .map(|interface| interface.id) + .collect(); + + diesel::delete(vlan_config_dsl::switch_vlan_interface_config) + .filter( + switch_vlan_interface_config::interface_config_id.eq_any( + interface_ids + ) ) + .execute_async(&conn) + .await?; + + // delete route configs + use db::schema::switch_port_settings_route_config; + use db::schema::switch_port_settings_route_config::dsl + as route_config_dsl; + + diesel::delete( + route_config_dsl::switch_port_settings_route_config ) + .filter(switch_port_settings_route_config::port_settings_id.eq(id)) .execute_async(&conn) .await?; - // delete route configs - use db::schema::switch_port_settings_route_config; - use db::schema::switch_port_settings_route_config::dsl - as route_config_dsl; + // delete bgp configurations + use db::schema::switch_port_settings_bgp_peer_config as bgp_peer; + use db::schema::switch_port_settings_bgp_peer_config::dsl + as bgp_peer_dsl; - diesel::delete( - route_config_dsl::switch_port_settings_route_config - ) - .filter(switch_port_settings_route_config::port_settings_id.eq(id)) - .execute_async(&conn) - .await?; + diesel::delete(bgp_peer_dsl::switch_port_settings_bgp_peer_config) + .filter(bgp_peer::port_settings_id.eq(id)) + .execute_async(&conn) + .await?; - // delete bgp configurations - use db::schema::switch_port_settings_bgp_peer_config as bgp_peer; - use db::schema::switch_port_settings_bgp_peer_config::dsl - as bgp_peer_dsl; + // delete address configs + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; - diesel::delete(bgp_peer_dsl::switch_port_settings_bgp_peer_config) - .filter(bgp_peer::port_settings_id.eq(id)) - .execute_async(&conn) + let port_settings_addrs = diesel::delete( + address_config_dsl::switch_port_settings_address_config, + ) + .filter(address_config::port_settings_id.eq(id)) + .returning(SwitchPortAddressConfig::as_returning()) + .get_results_async(&conn) .await?; - // delete address configs - use db::schema::switch_port_settings_address_config::{ - self as address_config, dsl as address_config_dsl, - }; - - let port_settings_addrs = diesel::delete( - address_config_dsl::switch_port_settings_address_config, - ) - .filter(address_config::port_settings_id.eq(id)) - .returning(SwitchPortAddressConfig::as_returning()) - .get_results_async(&conn) - .await?; + use db::schema::address_lot_rsvd_block::dsl as rsvd_block_dsl; - use db::schema::address_lot_rsvd_block::dsl as rsvd_block_dsl; + for ps in &port_settings_addrs { + diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) + .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) + .execute_async(&conn) + .await?; + } - for ps in &port_settings_addrs { - diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) - .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) - .execute_async(&conn) - .await?; + Ok(()) } - - Ok(()) }) .await - .map_err(|e| match e { - TxnError::CustomError( - SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound) => { - Error::invalid_request("port settings not found") + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SwitchPortSettingsDeleteError::SwitchPortSettingsNotFound => { + Error::invalid_request("port settings not found") + } + } + } else { + let name = match ¶ms.port_settings { + Some(name_or_id) => name_or_id.to_string(), + None => String::new(), + }; + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::SwitchPortSettings, + &name, + ), + ) } - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => { - let name = match ¶ms.port_settings { - Some(name_or_id) => name_or_id.to_string(), - None => String::new(), - }; - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::SwitchPortSettings, - &name, - ), - ) - }, - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, }) } @@ -666,174 +671,178 @@ impl DataStore { enum SwitchPortSettingsGetError { NotFound(external::Name), } - type TxnError = TransactionError; + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - // get the top level port settings object - use db::schema::switch_port_settings::{ - self, dsl as port_settings_dsl, - }; - - let id = match name_or_id { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name_str = name.to_string(); + self.transaction_retry_wrapper("switch_port_settings_get") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + // get the top level port settings object + use db::schema::switch_port_settings::{ + self, dsl as port_settings_dsl, + }; + + let id = match name_or_id { + NameOrId::Id(id) => *id, + NameOrId::Name(name) => { + let name_str = name.to_string(); + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::name.eq(name_str)) + .select(switch_port_settings::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or_else(diesel_error, |_| { + SwitchPortSettingsGetError::NotFound( + name.clone(), + ) + }) + })? + } + }; + + let settings: SwitchPortSettings = port_settings_dsl::switch_port_settings .filter(switch_port_settings::time_deleted.is_null()) - .filter(switch_port_settings::name.eq(name_str)) - .select(switch_port_settings::id) + .filter(switch_port_settings::id.eq(id)) + .select(SwitchPortSettings::as_select()) .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError( - SwitchPortSettingsGetError::NotFound( - name.clone(), - ), - ) - })? - } - }; - - let settings: SwitchPortSettings = - port_settings_dsl::switch_port_settings - .filter(switch_port_settings::time_deleted.is_null()) - .filter(switch_port_settings::id.eq(id)) - .select(SwitchPortSettings::as_select()) - .limit(1) - .first_async::(&conn) - .await?; + .first_async::(&conn) + .await?; - // get the port config - use db::schema::switch_port_settings_port_config::{ - self as port_config, dsl as port_config_dsl, - }; - let port: SwitchPortConfig = - port_config_dsl::switch_port_settings_port_config - .filter(port_config::port_settings_id.eq(id)) - .select(SwitchPortConfig::as_select()) - .limit(1) - .first_async::(&conn) - .await?; + // get the port config + use db::schema::switch_port_settings_port_config::{ + self as port_config, dsl as port_config_dsl, + }; + let port: SwitchPortConfig = + port_config_dsl::switch_port_settings_port_config + .filter(port_config::port_settings_id.eq(id)) + .select(SwitchPortConfig::as_select()) + .limit(1) + .first_async::(&conn) + .await?; - // initialize result - let mut result = - SwitchPortSettingsCombinedResult::new(settings, port); + // initialize result + let mut result = + SwitchPortSettingsCombinedResult::new(settings, port); - // get the link configs - use db::schema::switch_port_settings_link_config::{ - self as link_config, dsl as link_config_dsl, - }; + // get the link configs + use db::schema::switch_port_settings_link_config::{ + self as link_config, dsl as link_config_dsl, + }; - result.links = link_config_dsl::switch_port_settings_link_config - .filter(link_config::port_settings_id.eq(id)) - .select(SwitchPortLinkConfig::as_select()) - .load_async::(&conn) - .await?; + result.links = link_config_dsl::switch_port_settings_link_config + .filter(link_config::port_settings_id.eq(id)) + .select(SwitchPortLinkConfig::as_select()) + .load_async::(&conn) + .await?; - let lldp_svc_ids: Vec = result - .links - .iter() - .map(|link| link.lldp_service_config_id) - .collect(); - - use db::schema::lldp_service_config as lldp_config; - use db::schema::lldp_service_config::dsl as lldp_dsl; - result.link_lldp = lldp_dsl::lldp_service_config - .filter(lldp_config::id.eq_any(lldp_svc_ids)) - .select(LldpServiceConfig::as_select()) - .limit(1) - .load_async::(&conn) - .await?; + let lldp_svc_ids: Vec = result + .links + .iter() + .map(|link| link.lldp_service_config_id) + .collect(); + + use db::schema::lldp_service_config as lldp_config; + use db::schema::lldp_service_config::dsl as lldp_dsl; + result.link_lldp = lldp_dsl::lldp_service_config + .filter(lldp_config::id.eq_any(lldp_svc_ids)) + .select(LldpServiceConfig::as_select()) + .limit(1) + .load_async::(&conn) + .await?; - // get the interface configs - use db::schema::switch_port_settings_interface_config::{ - self as interface_config, dsl as interface_config_dsl, - }; + // get the interface configs + use db::schema::switch_port_settings_interface_config::{ + self as interface_config, dsl as interface_config_dsl, + }; - result.interfaces = - interface_config_dsl::switch_port_settings_interface_config - .filter(interface_config::port_settings_id.eq(id)) - .select(SwitchInterfaceConfig::as_select()) - .load_async::(&conn) + result.interfaces = + interface_config_dsl::switch_port_settings_interface_config + .filter(interface_config::port_settings_id.eq(id)) + .select(SwitchInterfaceConfig::as_select()) + .load_async::(&conn) + .await?; + + use db::schema::switch_vlan_interface_config as vlan_config; + use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; + let interface_ids: Vec = result + .interfaces + .iter() + .map(|interface| interface.id) + .collect(); + + result.vlan_interfaces = vlan_dsl::switch_vlan_interface_config + .filter(vlan_config::interface_config_id.eq_any(interface_ids)) + .select(SwitchVlanInterfaceConfig::as_select()) + .load_async::(&conn) .await?; - use db::schema::switch_vlan_interface_config as vlan_config; - use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; - let interface_ids: Vec = result - .interfaces - .iter() - .map(|interface| interface.id) - .collect(); - - result.vlan_interfaces = vlan_dsl::switch_vlan_interface_config - .filter(vlan_config::interface_config_id.eq_any(interface_ids)) - .select(SwitchVlanInterfaceConfig::as_select()) - .load_async::(&conn) - .await?; - - // get the route configs - use db::schema::switch_port_settings_route_config::{ - self as route_config, dsl as route_config_dsl, - }; + // get the route configs + use db::schema::switch_port_settings_route_config::{ + self as route_config, dsl as route_config_dsl, + }; - result.routes = route_config_dsl::switch_port_settings_route_config - .filter(route_config::port_settings_id.eq(id)) - .select(SwitchPortRouteConfig::as_select()) - .load_async::(&conn) - .await?; + result.routes = route_config_dsl::switch_port_settings_route_config + .filter(route_config::port_settings_id.eq(id)) + .select(SwitchPortRouteConfig::as_select()) + .load_async::(&conn) + .await?; - // get the bgp peer configs - use db::schema::switch_port_settings_bgp_peer_config::{ - self as bgp_peer, dsl as bgp_peer_dsl, - }; + // get the bgp peer configs + use db::schema::switch_port_settings_bgp_peer_config::{ + self as bgp_peer, dsl as bgp_peer_dsl, + }; - result.bgp_peers = - bgp_peer_dsl::switch_port_settings_bgp_peer_config - .filter(bgp_peer::port_settings_id.eq(id)) - .select(SwitchPortBgpPeerConfig::as_select()) - .load_async::(&conn) - .await?; + result.bgp_peers = + bgp_peer_dsl::switch_port_settings_bgp_peer_config + .filter(bgp_peer::port_settings_id.eq(id)) + .select(SwitchPortBgpPeerConfig::as_select()) + .load_async::(&conn) + .await?; - // get the address configs - use db::schema::switch_port_settings_address_config::{ - self as address_config, dsl as address_config_dsl, - }; + // get the address configs + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; - result.addresses = - address_config_dsl::switch_port_settings_address_config - .filter(address_config::port_settings_id.eq(id)) - .select(SwitchPortAddressConfig::as_select()) - .load_async::(&conn) - .await?; + result.addresses = + address_config_dsl::switch_port_settings_address_config + .filter(address_config::port_settings_id.eq(id)) + .select(SwitchPortAddressConfig::as_select()) + .load_async::(&conn) + .await?; - Ok(result) + Ok(result) + } }) .await - .map_err(|e| match e { - TxnError::CustomError(SwitchPortSettingsGetError::NotFound( - name, - )) => Error::not_found_by_name( - ResourceType::SwitchPortSettings, - &name, - ), - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => { - let name = name_or_id.to_string(); - public_error_from_diesel( - e, - ErrorHandler::Conflict( + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SwitchPortSettingsGetError::NotFound(name) => { + Error::not_found_by_name( ResourceType::SwitchPortSettings, &name, - ), - ) + ) + } } - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, + } else { + let name = name_or_id.to_string(); + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::SwitchPortSettings, + &name, + ), + ) + } }) } @@ -850,7 +859,8 @@ impl DataStore { enum SwitchPortCreateError { RackNotFound, } - type TxnError = TransactionError; + + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; let switch_port = SwitchPort::new( @@ -861,46 +871,59 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - use db::schema::rack; - use db::schema::rack::dsl as rack_dsl; - rack_dsl::rack - .filter(rack::id.eq(rack_id)) - .select(rack::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError(SwitchPortCreateError::RackNotFound) - })?; - - // insert switch port - use db::schema::switch_port::dsl as switch_port_dsl; - let db_switch_port: SwitchPort = - diesel::insert_into(switch_port_dsl::switch_port) - .values(switch_port) - .returning(SwitchPort::as_returning()) - .get_result_async(&conn) - .await?; + self.transaction_retry_wrapper("switch_port_create") + .transaction(&conn, |conn| { + let err = err.clone(); + let switch_port = switch_port.clone(); + async move { + use db::schema::rack; + use db::schema::rack::dsl as rack_dsl; + rack_dsl::rack + .filter(rack::id.eq(rack_id)) + .select(rack::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|e| { + err.bail_retryable_or( + e, + SwitchPortCreateError::RackNotFound, + ) + })?; - Ok(db_switch_port) - }) - .await - .map_err(|e| match e { - TxnError::CustomError(SwitchPortCreateError::RackNotFound) => { - Error::invalid_request("rack not found") - } - TxnError::Database(e) => match e { - DieselError::DatabaseError(_, _) => public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::SwitchPort, - &format!("{}/{}/{}", rack_id, &switch_location, &port,), - ), - ), - _ => public_error_from_diesel(e, ErrorHandler::Server), - }, - }) + // insert switch port + use db::schema::switch_port::dsl as switch_port_dsl; + let db_switch_port: SwitchPort = + diesel::insert_into(switch_port_dsl::switch_port) + .values(switch_port) + .returning(SwitchPort::as_returning()) + .get_result_async(&conn) + .await?; + + Ok(db_switch_port) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SwitchPortCreateError::RackNotFound => { + Error::invalid_request("rack not found") + } + } + } else { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::SwitchPort, + &format!( + "{}/{}/{}", + rack_id, &switch_location, &port, + ), + ), + ) + } + }) } pub async fn switch_port_delete( @@ -914,58 +937,75 @@ impl DataStore { NotFound, ActiveSettings, } - type TxnError = TransactionError; + + let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage - conn.transaction_async(|conn| async move { - use db::schema::switch_port; - use db::schema::switch_port::dsl as switch_port_dsl; - - let switch_location = params.switch_location.to_string(); - let port_name = portname.to_string(); - let port: SwitchPort = switch_port_dsl::switch_port - .filter(switch_port::rack_id.eq(params.rack_id)) - .filter( - switch_port::switch_location.eq(switch_location.clone()), - ) - .filter(switch_port::port_name.eq(port_name.clone())) - .select(SwitchPort::as_select()) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError(SwitchPortDeleteError::NotFound) - })?; - - if port.port_settings_id.is_some() { - return Err(TxnError::CustomError( - SwitchPortDeleteError::ActiveSettings, - )); - } + self.transaction_retry_wrapper("switch_port_delete") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + use db::schema::switch_port; + use db::schema::switch_port::dsl as switch_port_dsl; + + let switch_location = params.switch_location.to_string(); + let port_name = portname.to_string(); + let port: SwitchPort = switch_port_dsl::switch_port + .filter(switch_port::rack_id.eq(params.rack_id)) + .filter( + switch_port::switch_location + .eq(switch_location.clone()), + ) + .filter(switch_port::port_name.eq(port_name.clone())) + .select(SwitchPort::as_select()) + .limit(1) + .first_async::(&conn) + .await + .map_err(|diesel_error| { + err.bail_retryable_or( + diesel_error, + SwitchPortDeleteError::NotFound, + ) + })?; - diesel::delete(switch_port_dsl::switch_port) - .filter(switch_port::id.eq(port.id)) - .execute_async(&conn) - .await?; + if port.port_settings_id.is_some() { + return Err( + err.bail(SwitchPortDeleteError::ActiveSettings) + ); + } - Ok(()) - }) - .await - .map_err(|e| match e { - TxnError::CustomError(SwitchPortDeleteError::NotFound) => { - let name = &portname.clone(); - Error::not_found_by_name(ResourceType::SwitchPort, name) - } - TxnError::CustomError(SwitchPortDeleteError::ActiveSettings) => { - Error::invalid_request("must clear port settings first") - } - TxnError::Database(e) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - }) + diesel::delete(switch_port_dsl::switch_port) + .filter(switch_port::id.eq(port.id)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + SwitchPortDeleteError::NotFound => { + let name = &portname.clone(); + Error::not_found_by_name( + ResourceType::SwitchPort, + name, + ) + } + SwitchPortDeleteError::ActiveSettings => { + Error::invalid_request( + "must clear port settings first", + ) + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) } pub async fn switch_port_list( diff --git a/nexus/db-queries/src/db/datastore/update.rs b/nexus/db-queries/src/db/datastore/update.rs index 8b1eecb781..0790bd458e 100644 --- a/nexus/db-queries/src/db/datastore/update.rs +++ b/nexus/db-queries/src/db/datastore/update.rs @@ -8,15 +8,13 @@ use super::DataStore; use crate::authz; use crate::context::OpContext; use crate::db; -use crate::db::error::{ - public_error_from_diesel, ErrorHandler, TransactionError, -}; +use crate::db::error::{public_error_from_diesel, ErrorHandler}; use crate::db::model::{ ComponentUpdate, SemverVersion, SystemUpdate, UpdateArtifact, UpdateDeployment, UpdateStatus, UpdateableComponent, }; use crate::db::pagination::paginated; -use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; +use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use nexus_db_model::SystemUpdateComponentUpdate; @@ -141,36 +139,40 @@ impl DataStore { let version_string = update.version.to_string(); - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - let db_update = diesel::insert_into(component_update::table) - .values(update.clone()) - .returning(ComponentUpdate::as_returning()) - .get_result_async(&conn) - .await?; - - diesel::insert_into(join_table::table) - .values(SystemUpdateComponentUpdate { - system_update_id, - component_update_id: update.id(), - }) - .returning(SystemUpdateComponentUpdate::as_returning()) - .get_result_async(&conn) - .await?; - - Ok(db_update) + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("create_component_update") + .transaction(&conn, |conn| { + let update = update.clone(); + async move { + let db_update = + diesel::insert_into(component_update::table) + .values(update.clone()) + .returning(ComponentUpdate::as_returning()) + .get_result_async(&conn) + .await?; + + diesel::insert_into(join_table::table) + .values(SystemUpdateComponentUpdate { + system_update_id, + component_update_id: update.id(), + }) + .returning(SystemUpdateComponentUpdate::as_returning()) + .get_result_async(&conn) + .await?; + + Ok(db_update) + } }) .await - .map_err(|e| match e { - TransactionError::CustomError(e) => e, - TransactionError::Database(e) => public_error_from_diesel( + .map_err(|e| { + public_error_from_diesel( e, ErrorHandler::Conflict( ResourceType::ComponentUpdate, &version_string, ), - ), + ) }) } diff --git a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs index c5c2751723..230c3941ff 100644 --- a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs +++ b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs @@ -15,6 +15,7 @@ use crate::db::pool::DbConnection; use crate::db::queries::virtual_provisioning_collection_update::VirtualProvisioningCollectionUpdate; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; +use diesel::result::Error as DieselError; use omicron_common::api::external::{DeleteResult, Error}; use uuid::Uuid; @@ -52,13 +53,14 @@ impl DataStore { virtual_provisioning_collection, ) .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } pub(crate) async fn virtual_provisioning_collection_create_on_connection( &self, conn: &async_bb8_diesel::Connection, virtual_provisioning_collection: VirtualProvisioningCollection, - ) -> Result, Error> { + ) -> Result, DieselError> { use db::schema::virtual_provisioning_collection::dsl; let provisions: Vec = @@ -66,12 +68,10 @@ impl DataStore { .values(virtual_provisioning_collection) .on_conflict_do_nothing() .get_results_async(conn) - .await - .map_err(|e| { - public_error_from_diesel(e, ErrorHandler::Server) - })?; - self.virtual_provisioning_collection_producer - .append_all_metrics(&provisions)?; + .await?; + let _ = self + .virtual_provisioning_collection_producer + .append_all_metrics(&provisions); Ok(provisions) } @@ -103,16 +103,20 @@ impl DataStore { id: Uuid, ) -> DeleteResult { let conn = self.pool_connection_authorized(opctx).await?; - self.virtual_provisioning_collection_delete_on_connection(&conn, id) - .await + self.virtual_provisioning_collection_delete_on_connection( + &opctx.log, &conn, id, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } /// Delete a [`VirtualProvisioningCollection`] object. pub(crate) async fn virtual_provisioning_collection_delete_on_connection( &self, + log: &slog::Logger, conn: &async_bb8_diesel::Connection, id: Uuid, - ) -> DeleteResult { + ) -> Result<(), DieselError> { use db::schema::virtual_provisioning_collection::dsl; // NOTE: We don't really need to extract the value we're deleting from @@ -122,13 +126,11 @@ impl DataStore { .filter(dsl::id.eq(id)) .returning(VirtualProvisioningCollection::as_select()) .get_result_async(conn) - .await - .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + .await?; if !collection.is_empty() { - return Err(Error::internal_error(&format!( - "Collection deleted while non-empty: {collection:?}" - ))); + warn!(log, "Collection deleted while non-empty: {collection:?}"); + return Err(DieselError::RollbackTransaction); } Ok(()) } diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index 5f126050ae..4f31efd610 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -8,15 +8,14 @@ use super::DataStore; use crate::db; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::identity::Asset; use crate::db::model::Dataset; use crate::db::model::Region; use crate::db::model::RegionSnapshot; use crate::db::model::Volume; use crate::db::queries::volume::DecreaseCrucibleResourceCountAndSoftDeleteVolume; +use crate::transaction_retry::OptionalError; use anyhow::bail; -use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use diesel::prelude::*; use diesel::OptionalExtension; @@ -44,7 +43,6 @@ impl DataStore { #[error("Serde error during Volume creation: {0}")] SerdeError(#[from] serde_json::Error), } - type TxnError = TransactionError; // Grab all the targets that the volume construction request references. // Do this outside the transaction, as the data inside volume doesn't @@ -66,86 +64,91 @@ impl DataStore { crucible_targets }; - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - let maybe_volume: Option = dsl::volume - .filter(dsl::id.eq(volume.id())) - .select(Volume::as_select()) - .first_async(&conn) - .await - .optional() - .map_err(|e| { - TxnError::CustomError(VolumeCreationError::Public( - public_error_from_diesel(e, ErrorHandler::Server), - )) - })?; - - // If the volume existed already, return it and do not increase - // usage counts. - if let Some(volume) = maybe_volume { - return Ok(volume); - } - - // TODO do we need on_conflict do_nothing here? if the transaction - // model is read-committed, the SELECT above could return nothing, - // and the INSERT here could still result in a conflict. - // - // See also https://github.com/oxidecomputer/omicron/issues/1168 - let volume: Volume = diesel::insert_into(dsl::volume) - .values(volume.clone()) - .on_conflict(dsl::id) - .do_nothing() - .returning(Volume::as_returning()) - .get_result_async(&conn) - .await - .map_err(|e| { - TxnError::CustomError(VolumeCreationError::Public( - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::Volume, - volume.id().to_string().as_str(), - ), - ), - )) - })?; + let err = OptionalError::new(); + let conn = self.pool_connection_unauthorized().await?; + self.transaction_retry_wrapper("volume_create") + .transaction(&conn, |conn| { + let err = err.clone(); + let crucible_targets = crucible_targets.clone(); + let volume = volume.clone(); + async move { + let maybe_volume: Option = dsl::volume + .filter(dsl::id.eq(volume.id())) + .select(Volume::as_select()) + .first_async(&conn) + .await + .optional()?; - // Increase the usage count for Crucible resources according to the - // contents of the volume. + // If the volume existed already, return it and do not increase + // usage counts. + if let Some(volume) = maybe_volume { + return Ok(volume); + } - // Increase the number of uses for each referenced region snapshot. - use db::schema::region_snapshot::dsl as rs_dsl; - for read_only_target in &crucible_targets.read_only_targets { - diesel::update(rs_dsl::region_snapshot) - .filter( - rs_dsl::snapshot_addr.eq(read_only_target.clone()), - ) - .filter(rs_dsl::deleting.eq(false)) - .set( - rs_dsl::volume_references - .eq(rs_dsl::volume_references + 1), - ) - .execute_async(&conn) + // TODO do we need on_conflict do_nothing here? if the transaction + // model is read-committed, the SELECT above could return nothing, + // and the INSERT here could still result in a conflict. + // + // See also https://github.com/oxidecomputer/omicron/issues/1168 + let volume: Volume = diesel::insert_into(dsl::volume) + .values(volume.clone()) + .on_conflict(dsl::id) + .do_nothing() + .returning(Volume::as_returning()) + .get_result_async(&conn) .await .map_err(|e| { - TxnError::CustomError(VolumeCreationError::Public( - public_error_from_diesel( - e, - ErrorHandler::Server, - ), - )) + err.bail_retryable_or_else(e, |e| { + VolumeCreationError::Public( + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::Volume, + volume.id().to_string().as_str(), + ), + ), + ) + }) })?; - } - Ok(volume) + // Increase the usage count for Crucible resources according to the + // contents of the volume. + + // Increase the number of uses for each referenced region snapshot. + use db::schema::region_snapshot::dsl as rs_dsl; + for read_only_target in &crucible_targets.read_only_targets + { + diesel::update(rs_dsl::region_snapshot) + .filter( + rs_dsl::snapshot_addr + .eq(read_only_target.clone()), + ) + .filter(rs_dsl::deleting.eq(false)) + .set( + rs_dsl::volume_references + .eq(rs_dsl::volume_references + 1), + ) + .execute_async(&conn) + .await?; + } + + Ok(volume) + } }) .await - .map_err(|e| match e { - TxnError::CustomError(VolumeCreationError::Public(e)) => e, - - _ => { - Error::internal_error(&format!("Transaction error: {}", e)) + .map_err(|e| { + if let Some(err) = err.take() { + match err { + VolumeCreationError::Public(err) => err, + VolumeCreationError::SerdeError(err) => { + Error::internal_error(&format!( + "Transaction error: {}", + err + )) + } + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) } }) } @@ -192,16 +195,12 @@ impl DataStore { #[derive(Debug, thiserror::Error)] enum VolumeGetError { - #[error("Error during volume_checkout: {0}")] - DieselError(#[from] diesel::result::Error), - #[error("Serde error during volume_checkout: {0}")] SerdeError(#[from] serde_json::Error), #[error("Updated {0} database rows, expected {1}")] UnexpectedDatabaseUpdate(usize, usize), } - type TxnError = TransactionError; // We perform a transaction here, to be sure that on completion // of this, the database contains an updated version of the @@ -209,141 +208,141 @@ impl DataStore { // types that require it). The generation number (along with the // rest of the volume data) that was in the database is what is // returned to the caller. - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - // Grab the volume in question. - let volume = dsl::volume - .filter(dsl::id.eq(volume_id)) - .select(Volume::as_select()) - .get_result_async(&conn) - .await?; - - // Turn the volume.data into the VolumeConstructionRequest - let vcr: VolumeConstructionRequest = - serde_json::from_str(volume.data()).map_err(|e| { - TxnError::CustomError(VolumeGetError::SerdeError(e)) - })?; - - // Look to see if the VCR is a Volume type, and if so, look at - // its sub_volumes. If they are of type Region, then we need - // to update their generation numbers and record that update - // back to the database. We return to the caller whatever the - // original volume data was we pulled from the database. - match vcr { - VolumeConstructionRequest::Volume { - id, - block_size, - sub_volumes, - read_only_parent, - } => { - let mut update_needed = false; - let mut new_sv = Vec::new(); - for sv in sub_volumes { - match sv { - VolumeConstructionRequest::Region { - block_size, - blocks_per_extent, - extent_count, - opts, - gen, - } => { - update_needed = true; - new_sv.push( - VolumeConstructionRequest::Region { - block_size, - blocks_per_extent, - extent_count, - opts, - gen: gen + 1, - }, - ); - } - _ => { - new_sv.push(sv); + let err = OptionalError::new(); + let conn = self.pool_connection_unauthorized().await?; + + self.transaction_retry_wrapper("volume_checkout") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + // Grab the volume in question. + let volume = dsl::volume + .filter(dsl::id.eq(volume_id)) + .select(Volume::as_select()) + .get_result_async(&conn) + .await?; + + // Turn the volume.data into the VolumeConstructionRequest + let vcr: VolumeConstructionRequest = + serde_json::from_str(volume.data()).map_err(|e| { + err.bail(VolumeGetError::SerdeError(e)) + })?; + + // Look to see if the VCR is a Volume type, and if so, look at + // its sub_volumes. If they are of type Region, then we need + // to update their generation numbers and record that update + // back to the database. We return to the caller whatever the + // original volume data was we pulled from the database. + match vcr { + VolumeConstructionRequest::Volume { + id, + block_size, + sub_volumes, + read_only_parent, + } => { + let mut update_needed = false; + let mut new_sv = Vec::new(); + for sv in sub_volumes { + match sv { + VolumeConstructionRequest::Region { + block_size, + blocks_per_extent, + extent_count, + opts, + gen, + } => { + update_needed = true; + new_sv.push( + VolumeConstructionRequest::Region { + block_size, + blocks_per_extent, + extent_count, + opts, + gen: gen + 1, + }, + ); + } + _ => { + new_sv.push(sv); + } } } - } - // Only update the volume data if we found the type - // of volume that needed it. - if update_needed { - // Create a new VCR and fill in the contents - // from what the original volume had, but with our - // updated sub_volume records. - let new_vcr = VolumeConstructionRequest::Volume { - id, - block_size, - sub_volumes: new_sv, - read_only_parent, - }; - - let new_volume_data = serde_json::to_string( - &new_vcr, - ) - .map_err(|e| { - TxnError::CustomError( - VolumeGetError::SerdeError(e), - ) - })?; + // Only update the volume data if we found the type + // of volume that needed it. + if update_needed { + // Create a new VCR and fill in the contents + // from what the original volume had, but with our + // updated sub_volume records. + let new_vcr = VolumeConstructionRequest::Volume { + id, + block_size, + sub_volumes: new_sv, + read_only_parent, + }; - // Update the original volume_id with the new - // volume.data. - use db::schema::volume::dsl as volume_dsl; - let num_updated = - diesel::update(volume_dsl::volume) - .filter(volume_dsl::id.eq(volume_id)) - .set(volume_dsl::data.eq(new_volume_data)) - .execute_async(&conn) - .await?; + let new_volume_data = serde_json::to_string( + &new_vcr, + ) + .map_err(|e| { + err.bail(VolumeGetError::SerdeError(e)) + })?; - // This should update just one row. If it does - // not, then something is terribly wrong in the - // database. - if num_updated != 1 { - return Err(TxnError::CustomError( - VolumeGetError::UnexpectedDatabaseUpdate( - num_updated, - 1, - ), - )); + // Update the original volume_id with the new + // volume.data. + use db::schema::volume::dsl as volume_dsl; + let num_updated = + diesel::update(volume_dsl::volume) + .filter(volume_dsl::id.eq(volume_id)) + .set(volume_dsl::data.eq(new_volume_data)) + .execute_async(&conn) + .await?; + + // This should update just one row. If it does + // not, then something is terribly wrong in the + // database. + if num_updated != 1 { + return Err(err.bail( + VolumeGetError::UnexpectedDatabaseUpdate( + num_updated, + 1, + ), + )); + } } } + VolumeConstructionRequest::Region { + block_size: _, + blocks_per_extent: _, + extent_count: _, + opts: _, + gen: _, + } => { + // We don't support a pure Region VCR at the volume + // level in the database, so this choice should + // never be encountered, but I want to know if it is. + panic!("Region not supported as a top level volume"); + } + VolumeConstructionRequest::File { + id: _, + block_size: _, + path: _, + } + | VolumeConstructionRequest::Url { + id: _, + block_size: _, + url: _, + } => {} } - VolumeConstructionRequest::Region { - block_size: _, - blocks_per_extent: _, - extent_count: _, - opts: _, - gen: _, - } => { - // We don't support a pure Region VCR at the volume - // level in the database, so this choice should - // never be encountered, but I want to know if it is. - panic!("Region not supported as a top level volume"); - } - VolumeConstructionRequest::File { - id: _, - block_size: _, - path: _, - } - | VolumeConstructionRequest::Url { - id: _, - block_size: _, - url: _, - } => {} + Ok(volume) } - Ok(volume) }) .await - .map_err(|e| match e { - TxnError::CustomError(VolumeGetError::DieselError(e)) => { - public_error_from_diesel(e, ErrorHandler::Server) - } - - _ => { - Error::internal_error(&format!("Transaction error: {}", e)) + .map_err(|e| { + if let Some(err) = err.take() { + return Error::internal_error(&format!("Transaction error: {}", err)); } + public_error_from_diesel(e, ErrorHandler::Server) }) } @@ -638,16 +637,12 @@ impl DataStore { ) -> Result { #[derive(Debug, thiserror::Error)] enum RemoveReadOnlyParentError { - #[error("Error removing read only parent: {0}")] - DieselError(#[from] diesel::result::Error), - #[error("Serde error removing read only parent: {0}")] SerdeError(#[from] serde_json::Error), #[error("Updated {0} database rows, expected {1}")] UnexpectedDatabaseUpdate(usize, usize), } - type TxnError = TransactionError; // In this single transaction: // - Get the given volume from the volume_id from the database @@ -663,170 +658,160 @@ impl DataStore { // data from original volume_id. // - Put the new temp VCR into the temp volume.data, update the // temp_volume in the database. - self.pool_connection_unauthorized() - .await? - .transaction_async(|conn| async move { - // Grab the volume in question. If the volume record was already - // deleted then we can just return. - let volume = { - use db::schema::volume::dsl; - - let volume = dsl::volume - .filter(dsl::id.eq(volume_id)) - .select(Volume::as_select()) - .get_result_async(&conn) - .await - .optional()?; - - let volume = if let Some(v) = volume { - v - } else { - // the volume does not exist, nothing to do. - return Ok(false); + let err = OptionalError::new(); + let conn = self.pool_connection_unauthorized().await?; + self.transaction_retry_wrapper("volume_remove_rop") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + // Grab the volume in question. If the volume record was already + // deleted then we can just return. + let volume = { + use db::schema::volume::dsl; + + let volume = dsl::volume + .filter(dsl::id.eq(volume_id)) + .select(Volume::as_select()) + .get_result_async(&conn) + .await + .optional()?; + + let volume = if let Some(v) = volume { + v + } else { + // the volume does not exist, nothing to do. + return Ok(false); + }; + + if volume.time_deleted.is_some() { + // this volume is deleted, so let whatever is deleting + // it clean it up. + return Ok(false); + } else { + // A volume record exists, and was not deleted, we + // can attempt to remove its read_only_parent. + volume + } }; - if volume.time_deleted.is_some() { - // this volume is deleted, so let whatever is deleting - // it clean it up. - return Ok(false); - } else { - // A volume record exists, and was not deleted, we - // can attempt to remove its read_only_parent. - volume - } - }; - - // If a read_only_parent exists, remove it from volume_id, and - // attach it to temp_volume_id. - let vcr: VolumeConstructionRequest = - serde_json::from_str( - volume.data() - ) - .map_err(|e| { - TxnError::CustomError( - RemoveReadOnlyParentError::SerdeError( - e, - ), + // If a read_only_parent exists, remove it from volume_id, and + // attach it to temp_volume_id. + let vcr: VolumeConstructionRequest = + serde_json::from_str( + volume.data() ) - })?; - - match vcr { - VolumeConstructionRequest::Volume { - id, - block_size, - sub_volumes, - read_only_parent, - } => { - if read_only_parent.is_none() { - // This volume has no read_only_parent - Ok(false) - } else { - // Create a new VCR and fill in the contents - // from what the original volume had. - let new_vcr = VolumeConstructionRequest::Volume { - id, - block_size, - sub_volumes, - read_only_parent: None, - }; - - let new_volume_data = - serde_json::to_string( - &new_vcr + .map_err(|e| { + err.bail( + RemoveReadOnlyParentError::SerdeError( + e, ) - .map_err(|e| { - TxnError::CustomError( - RemoveReadOnlyParentError::SerdeError( - e, - ), - ) - })?; + ) + })?; - // Update the original volume_id with the new - // volume.data. - use db::schema::volume::dsl as volume_dsl; - let num_updated = diesel::update(volume_dsl::volume) - .filter(volume_dsl::id.eq(volume_id)) - .set(volume_dsl::data.eq(new_volume_data)) - .execute_async(&conn) - .await?; - - // This should update just one row. If it does - // not, then something is terribly wrong in the - // database. - if num_updated != 1 { - return Err(TxnError::CustomError( - RemoveReadOnlyParentError::UnexpectedDatabaseUpdate(num_updated, 1), - )); - } + match vcr { + VolumeConstructionRequest::Volume { + id, + block_size, + sub_volumes, + read_only_parent, + } => { + if read_only_parent.is_none() { + // This volume has no read_only_parent + Ok(false) + } else { + // Create a new VCR and fill in the contents + // from what the original volume had. + let new_vcr = VolumeConstructionRequest::Volume { + id, + block_size, + sub_volumes, + read_only_parent: None, + }; - // Make a new VCR, with the information from - // our temp_volume_id, but the read_only_parent - // from the original volume. - let rop_vcr = VolumeConstructionRequest::Volume { - id: temp_volume_id, - block_size, - sub_volumes: vec![], - read_only_parent, - }; - let rop_volume_data = - serde_json::to_string( - &rop_vcr - ) - .map_err(|e| { - TxnError::CustomError( - RemoveReadOnlyParentError::SerdeError( - e, - ), + let new_volume_data = + serde_json::to_string( + &new_vcr ) - })?; - // Update the temp_volume_id with the volume - // data that contains the read_only_parent. - let num_updated = - diesel::update(volume_dsl::volume) - .filter(volume_dsl::id.eq(temp_volume_id)) - .filter(volume_dsl::time_deleted.is_null()) - .set(volume_dsl::data.eq(rop_volume_data)) + .map_err(|e| { + err.bail(RemoveReadOnlyParentError::SerdeError( + e, + )) + })?; + + // Update the original volume_id with the new + // volume.data. + use db::schema::volume::dsl as volume_dsl; + let num_updated = diesel::update(volume_dsl::volume) + .filter(volume_dsl::id.eq(volume_id)) + .set(volume_dsl::data.eq(new_volume_data)) .execute_async(&conn) .await?; - if num_updated != 1 { - return Err(TxnError::CustomError( - RemoveReadOnlyParentError::UnexpectedDatabaseUpdate(num_updated, 1), - )); + + // This should update just one row. If it does + // not, then something is terribly wrong in the + // database. + if num_updated != 1 { + return Err(err.bail(RemoveReadOnlyParentError::UnexpectedDatabaseUpdate(num_updated, 1))); + } + + // Make a new VCR, with the information from + // our temp_volume_id, but the read_only_parent + // from the original volume. + let rop_vcr = VolumeConstructionRequest::Volume { + id: temp_volume_id, + block_size, + sub_volumes: vec![], + read_only_parent, + }; + let rop_volume_data = + serde_json::to_string( + &rop_vcr + ) + .map_err(|e| { + err.bail(RemoveReadOnlyParentError::SerdeError( + e, + )) + })?; + // Update the temp_volume_id with the volume + // data that contains the read_only_parent. + let num_updated = + diesel::update(volume_dsl::volume) + .filter(volume_dsl::id.eq(temp_volume_id)) + .filter(volume_dsl::time_deleted.is_null()) + .set(volume_dsl::data.eq(rop_volume_data)) + .execute_async(&conn) + .await?; + if num_updated != 1 { + return Err(err.bail(RemoveReadOnlyParentError::UnexpectedDatabaseUpdate(num_updated, 1))); + } + Ok(true) } - Ok(true) } - } - VolumeConstructionRequest::File { id: _, block_size: _, path: _ } - | VolumeConstructionRequest::Region { - block_size: _, - blocks_per_extent: _, - extent_count: _, - opts: _, - gen: _ } - | VolumeConstructionRequest::Url { id: _, block_size: _, url: _ } => { - // Volume has a format that does not contain ROPs - Ok(false) + VolumeConstructionRequest::File { id: _, block_size: _, path: _ } + | VolumeConstructionRequest::Region { + block_size: _, + blocks_per_extent: _, + extent_count: _, + opts: _, + gen: _ } + | VolumeConstructionRequest::Url { id: _, block_size: _, url: _ } => { + // Volume has a format that does not contain ROPs + Ok(false) + } } } }) .await - .map_err(|e| match e { - TxnError::CustomError( - RemoveReadOnlyParentError::DieselError(e), - ) => public_error_from_diesel( - e, - ErrorHandler::Server, - ), - - _ => { - Error::internal_error(&format!("Transaction error: {}", e)) + .map_err(|e| { + if let Some(err) = err.take() { + return Error::internal_error(&format!("Transaction error: {}", err)); } + public_error_from_diesel(e, ErrorHandler::Server) }) } } -#[derive(Default, Debug, Serialize, Deserialize)] +#[derive(Default, Clone, Debug, Serialize, Deserialize)] pub struct CrucibleTargets { pub read_only_targets: Vec, } diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 6db99465a3..069ce63028 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -12,7 +12,6 @@ use crate::db::collection_insert::AsyncInsertError; use crate::db::collection_insert::DatastoreCollection; use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; -use crate::db::error::TransactionError; use crate::db::fixed_data::vpc::SERVICES_VPC_ID; use crate::db::identity::Resource; use crate::db::model::IncompleteVpc; @@ -37,7 +36,7 @@ use crate::db::queries::vpc::InsertVpcQuery; use crate::db::queries::vpc::VniSearchIter; use crate::db::queries::vpc_subnet::FilterConflictingVpcSubnetRangesQuery; use crate::db::queries::vpc_subnet::SubnetError; -use async_bb8_diesel::AsyncConnection; +use crate::transaction_retry::OptionalError; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; @@ -580,53 +579,65 @@ impl DataStore { .set(dsl::time_deleted.eq(now)); let rules_is_empty = rules.is_empty(); - let insert_new_query = Vpc::insert_resource( - authz_vpc.id(), - diesel::insert_into(dsl::vpc_firewall_rule).values(rules), - ); - #[derive(Debug)] enum FirewallUpdateError { CollectionNotFound, } - type TxnError = TransactionError; + + let err = OptionalError::new(); // TODO-scalability: Ideally this would be a CTE so we don't need to // hold a transaction open across multiple roundtrips from the database, // but for now we're using a transaction due to the severely decreased // legibility of CTEs via diesel right now. - self.pool_connection_authorized(opctx) - .await? - .transaction_async(|conn| async move { - delete_old_query.execute_async(&conn).await?; - - // The generation count update on the vpc table row will take a - // write lock on the row, ensuring that the vpc was not deleted - // concurently. - if rules_is_empty { - return Ok(vec![]); - } - insert_new_query + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("vpc_update_firewall_rules") + .transaction(&conn, |conn| { + let err = err.clone(); + let delete_old_query = delete_old_query.clone(); + let rules = rules.clone(); + async move { + delete_old_query.execute_async(&conn).await?; + + // The generation count update on the vpc table row will take a + // write lock on the row, ensuring that the vpc was not deleted + // concurently. + if rules_is_empty { + return Ok(vec![]); + } + Vpc::insert_resource( + authz_vpc.id(), + diesel::insert_into(dsl::vpc_firewall_rule) + .values(rules), + ) .insert_and_get_results_async(&conn) .await .map_err(|e| match e { AsyncInsertError::CollectionNotFound => { - TxnError::CustomError( - FirewallUpdateError::CollectionNotFound, - ) + err.bail(FirewallUpdateError::CollectionNotFound) } - AsyncInsertError::DatabaseError(e) => e.into(), + AsyncInsertError::DatabaseError(e) => e, }) + } }) .await - .map_err(|e| match e { - TxnError::CustomError( - FirewallUpdateError::CollectionNotFound, - ) => Error::not_found_by_id(ResourceType::Vpc, &authz_vpc.id()), - TxnError::Database(e) => public_error_from_diesel( - e, - ErrorHandler::NotFoundByResource(authz_vpc), - ), + .map_err(|e| { + if let Some(err) = err.take() { + match err { + FirewallUpdateError::CollectionNotFound => { + Error::not_found_by_id( + ResourceType::Vpc, + &authz_vpc.id(), + ) + } + } + } else { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_vpc), + ) + } }) } diff --git a/nexus/db-queries/src/db/error.rs b/nexus/db-queries/src/db/error.rs index cbe2b0a71f..fc7f30da93 100644 --- a/nexus/db-queries/src/db/error.rs +++ b/nexus/db-queries/src/db/error.rs @@ -17,7 +17,7 @@ pub enum TransactionError { /// The customizable error type. /// /// This error should be used for all non-Diesel transaction failures. - #[error("Custom transaction error; {0}")] + #[error("Custom transaction error: {0}")] CustomError(T), /// The Diesel error type. @@ -28,31 +28,61 @@ pub enum TransactionError { Database(#[from] DieselError), } +pub fn retryable(error: &DieselError) -> bool { + match error { + DieselError::DatabaseError(kind, boxed_error_information) => match kind + { + DieselErrorKind::SerializationFailure => { + return boxed_error_information + .message() + .starts_with("restart transaction"); + } + _ => false, + }, + _ => false, + } +} + +/// Identifies if the error is retryable or not. +pub enum MaybeRetryable { + /// The error isn't retryable. + NotRetryable(T), + /// The error is retryable. + Retryable(DieselError), +} + +impl TransactionError { + /// Identifies that the error could be returned from a Diesel transaction. + /// + /// Allows callers to propagate arbitrary errors out of transaction contexts + /// without losing information that might be valuable to the calling context, + /// such as "does this particular error indicate that the entire transaction + /// should retry?". + pub fn retryable(self) -> MaybeRetryable { + use MaybeRetryable::*; + + match self { + TransactionError::Database(err) if retryable(&err) => { + Retryable(err) + } + _ => NotRetryable(self), + } + } +} + impl From for TransactionError { fn from(err: PublicError) -> Self { TransactionError::CustomError(err) } } -impl TransactionError { - /// Based on [the CRDB][1] docs, return true if this transaction must be - /// retried. - /// - /// [1]: https://www.cockroachlabs.com/docs/v23.1/transaction-retry-error-reference#client-side-retry-handling - pub fn retry_transaction(&self) -> bool { - match &self { - Self::Database(DieselError::DatabaseError( - kind, - boxed_error_information, - )) => match kind { - DieselErrorKind::SerializationFailure => { - return boxed_error_information - .message() - .starts_with("restart transaction"); - } - _ => false, - }, - _ => false, +impl From> for PublicError { + fn from(err: TransactionError) -> Self { + match err { + TransactionError::CustomError(err) => err, + TransactionError::Database(err) => { + public_error_from_diesel(err, ErrorHandler::Server) + } } } } diff --git a/nexus/db-queries/src/db/mod.rs b/nexus/db-queries/src/db/mod.rs index b7c7079b54..e6b8743e94 100644 --- a/nexus/db-queries/src/db/mod.rs +++ b/nexus/db-queries/src/db/mod.rs @@ -17,7 +17,7 @@ mod config; mod cte_utils; // This is marked public for use by the integration tests pub mod datastore; -mod error; +pub(crate) mod error; mod explain; pub mod fixed_data; pub mod lookup; @@ -42,7 +42,7 @@ pub use nexus_db_model::schema; pub use crate::db::error::TransactionError; pub use config::Config; pub use datastore::DataStore; -pub use pool::Pool; +pub use pool::{DbConnection, Pool}; pub use saga_recovery::{recover, CompletionTask, RecoveryTask}; pub use saga_types::SecId; pub use sec_store::CockroachDbSecStore; diff --git a/nexus/db-queries/src/db/queries/network_interface.rs b/nexus/db-queries/src/db/queries/network_interface.rs index 84a81a7b7a..1dbe57da6f 100644 --- a/nexus/db-queries/src/db/queries/network_interface.rs +++ b/nexus/db-queries/src/db/queries/network_interface.rs @@ -5,6 +5,7 @@ //! Queries for inserting and deleting network interfaces. use crate::db; +use crate::db::error::{public_error_from_diesel, retryable, ErrorHandler}; use crate::db::model::IncompleteNetworkInterface; use crate::db::pool::DbConnection; use crate::db::queries::next_item::DefaultShiftGenerator; @@ -120,6 +121,8 @@ pub enum InsertError { InstanceMustBeStopped(Uuid), /// The instance does not exist at all, or is in the destroyed state. InstanceNotFound(Uuid), + /// The operation occurred within a transaction, and is retryable + Retryable(DieselError), /// Any other error External(external::Error), } @@ -135,7 +138,6 @@ impl InsertError { e: DieselError, interface: &IncompleteNetworkInterface, ) -> Self { - use crate::db::error; match e { // Catch the specific errors designed to communicate the failures we // want to distinguish @@ -143,9 +145,9 @@ impl InsertError { decode_database_error(e, interface) } // Any other error at all is a bug - _ => InsertError::External(error::public_error_from_diesel( + _ => InsertError::External(public_error_from_diesel( e, - error::ErrorHandler::Server, + ErrorHandler::Server, )), } } @@ -209,6 +211,9 @@ impl InsertError { InsertError::InstanceNotFound(id) => { external::Error::not_found_by_id(external::ResourceType::Instance, &id) } + InsertError::Retryable(err) => { + public_error_from_diesel(err, ErrorHandler::Server) + } InsertError::External(e) => e, } } @@ -290,6 +295,10 @@ fn decode_database_error( r#"uuid: incorrect UUID length: non-unique-subnets"#, ); + if retryable(&err) { + return InsertError::Retryable(err); + } + match err { // If the address allocation subquery fails, we'll attempt to insert // NULL for the `ip` column. This checks that the non-NULL constraint on diff --git a/nexus/db-queries/src/lib.rs b/nexus/db-queries/src/lib.rs index a693f7ff42..5d1927ebc7 100644 --- a/nexus/db-queries/src/lib.rs +++ b/nexus/db-queries/src/lib.rs @@ -9,6 +9,7 @@ pub mod authz; pub mod context; pub mod db; pub mod provisioning; +pub mod transaction_retry; #[macro_use] extern crate slog; diff --git a/nexus/db-queries/src/transaction_retry.rs b/nexus/db-queries/src/transaction_retry.rs new file mode 100644 index 0000000000..c474b729f8 --- /dev/null +++ b/nexus/db-queries/src/transaction_retry.rs @@ -0,0 +1,341 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Helper types for performing automatic transaction retries + +use async_bb8_diesel::AsyncConnection; +use chrono::Utc; +use diesel::result::Error as DieselError; +use oximeter::{types::Sample, Metric, MetricsError, Target}; +use rand::{thread_rng, Rng}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +// Identifies "which" transaction is retrying +#[derive(Debug, Clone, Target)] +struct DatabaseTransaction { + name: String, +} + +// Identifies that a retry has occurred, and track how long +// the transaction took (either since starting, or since the last +// retry failure was recorded). +#[derive(Debug, Clone, Metric)] +struct RetryData { + #[datum] + latency: f64, + attempt: u32, +} + +// Collects all transaction retry samples +#[derive(Debug, Default, Clone)] +pub(crate) struct Producer { + samples: Arc>>, +} + +impl Producer { + pub(crate) fn new() -> Self { + Self { samples: Arc::new(Mutex::new(vec![])) } + } + + fn append( + &self, + transaction: &DatabaseTransaction, + data: &RetryData, + ) -> Result<(), MetricsError> { + let sample = Sample::new_with_timestamp(Utc::now(), transaction, data)?; + self.samples.lock().unwrap().push(sample); + Ok(()) + } +} + +struct RetryHelperInner { + start: chrono::DateTime, + attempts: u32, +} + +impl RetryHelperInner { + fn new() -> Self { + Self { start: Utc::now(), attempts: 1 } + } + + fn tick(&mut self) -> Self { + let start = self.start; + let attempts = self.attempts; + + self.start = Utc::now(); + self.attempts += 1; + + Self { start, attempts } + } +} + +/// Helper utility for tracking retry attempts and latency. +/// Intended to be used from within "transaction_async_with_retry". +pub struct RetryHelper { + producer: Producer, + name: &'static str, + inner: Mutex, +} + +const MIN_RETRY_BACKOFF: Duration = Duration::from_millis(0); +const MAX_RETRY_BACKOFF: Duration = Duration::from_millis(50); +const MAX_RETRY_ATTEMPTS: u32 = 10; + +impl RetryHelper { + /// Creates a new RetryHelper, and starts a timer tracking the transaction + /// duration. + pub(crate) fn new(producer: &Producer, name: &'static str) -> Self { + Self { + producer: producer.clone(), + name, + inner: Mutex::new(RetryHelperInner::new()), + } + } + + /// Calls the function "f" in an asynchronous, retryable transaction. + pub async fn transaction( + self, + conn: &async_bb8_diesel::Connection, + f: Func, + ) -> Result + where + R: Send + 'static, + Fut: std::future::Future> + Send, + Func: Fn(async_bb8_diesel::Connection) -> Fut + + Send + + Sync, + { + conn.transaction_async_with_retry(f, self.as_callback()).await + } + + // Called upon retryable transaction failure. + // + // This function: + // - Appends a metric identifying the duration of the transaction operation + // - Performs a random (uniform) backoff (limited to less than 50 ms) + // - Returns "true" if the transaction should be restarted + async fn retry_callback(&self) -> bool { + // Look at the current attempt and start time so we can log this + // information before we start sleeping. + let (start, attempt) = { + let inner = self.inner.lock().unwrap(); + (inner.start, inner.attempts) + }; + + let latency = (Utc::now() - start) + .to_std() + .unwrap_or(Duration::ZERO) + .as_secs_f64(); + + let _ = self.producer.append( + &DatabaseTransaction { name: self.name.into() }, + &RetryData { latency, attempt }, + ); + + // This backoff is not exponential, but I'm not sure we actually want + // that much backoff here. If we're repeatedly failing, it would + // probably be better to fail the operation, at which point Oximeter + // will keep track of the failing transaction and identify that it's a + // high-priority target for CTE conversion. + let duration = { + let mut rng = thread_rng(); + rng.gen_range(MIN_RETRY_BACKOFF..MAX_RETRY_BACKOFF) + }; + tokio::time::sleep(duration).await; + + // Now that we've finished sleeping, reset the timer and bump the number + // of attempts we've tried. + let inner = self.inner.lock().unwrap().tick(); + return inner.attempts < MAX_RETRY_ATTEMPTS; + } + + /// Converts this function to a retryable callback that can be used from + /// "transaction_async_with_retry". + pub(crate) fn as_callback( + self, + ) -> impl Fn() -> futures::future::BoxFuture<'static, bool> { + let r = Arc::new(self); + move || { + let r = r.clone(); + Box::pin(async move { r.retry_callback().await }) + } + } +} + +impl oximeter::Producer for Producer { + fn produce( + &mut self, + ) -> Result + 'static>, MetricsError> { + let samples = std::mem::take(&mut *self.samples.lock().unwrap()); + Ok(Box::new(samples.into_iter())) + } +} + +/// Helper utility for passing non-retryable errors out-of-band from +/// transactions. +/// +/// Transactions prefer to act on the `diesel::result::Error` type, +/// but transaction users may want more meaningful error types. +/// This utility helps callers safely propagate back Diesel errors while +/// retaining auxiliary error info. +pub struct OptionalError(Arc>>); + +impl Clone for OptionalError { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl OptionalError { + pub fn new() -> Self { + Self(Arc::new(Mutex::new(None))) + } + + /// Sets "Self" to the value of `error` and returns `DieselError::RollbackTransaction`. + pub fn bail(&self, err: E) -> DieselError { + (*self.0.lock().unwrap()).replace(err); + DieselError::RollbackTransaction + } + + /// If `diesel_error` is retryable, returns it without setting Self. + /// + /// Otherwise, sets "Self" to the value of `err`, and returns + /// `DieselError::RollbackTransaction`. + pub fn bail_retryable_or( + &self, + diesel_error: DieselError, + err: E, + ) -> DieselError { + self.bail_retryable_or_else(diesel_error, |_diesel_error| err) + } + + /// If `diesel_error` is retryable, returns it without setting Self. + /// + /// Otherwise, sets "Self" to the value of `f` applied to `diesel_err`, and + /// returns `DieselError::RollbackTransaction`. + pub fn bail_retryable_or_else( + &self, + diesel_error: DieselError, + f: F, + ) -> DieselError + where + F: FnOnce(DieselError) -> E, + { + if crate::db::error::retryable(&diesel_error) { + return diesel_error; + } else { + self.bail(f(diesel_error)) + } + } + + /// If "Self" was previously set to a non-retryable error, return it. + pub fn take(self) -> Option { + (*self.0.lock().unwrap()).take() + } +} + +#[cfg(test)] +mod test { + use super::*; + + use crate::db::datastore::datastore_test; + use nexus_test_utils::db::test_setup_database; + use omicron_test_utils::dev; + use oximeter::types::FieldValue; + + // If a transaction is explicitly rolled back, we should not expect any + // samples to be taken. With no retries, this is just a normal operation + // failure. + #[tokio::test] + async fn test_transaction_rollback_produces_no_samples() { + let logctx = dev::test_setup_log( + "test_transaction_rollback_produces_no_samples", + ); + let mut db = test_setup_database(&logctx.log).await; + let (_opctx, datastore) = datastore_test(&logctx, &db).await; + + let conn = datastore.pool_connection_for_tests().await.unwrap(); + + datastore + .transaction_retry_wrapper( + "test_transaction_rollback_produces_no_samples", + ) + .transaction(&conn, |_conn| async move { + Err::<(), _>(diesel::result::Error::RollbackTransaction) + }) + .await + .expect_err("Should have failed"); + + let samples = datastore + .transaction_retry_producer() + .samples + .lock() + .unwrap() + .clone(); + assert_eq!(samples, vec![]); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + + // If a transaction fails with a retryable error, we record samples, + // providing oximeter-level information about the attempts. + #[tokio::test] + async fn test_transaction_retry_produces_samples() { + let logctx = + dev::test_setup_log("test_transaction_retry_produces_samples"); + let mut db = test_setup_database(&logctx.log).await; + let (_opctx, datastore) = datastore_test(&logctx, &db).await; + + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore + .transaction_retry_wrapper( + "test_transaction_retry_produces_samples", + ) + .transaction(&conn, |_conn| async move { + Err::<(), _>(diesel::result::Error::DatabaseError( + diesel::result::DatabaseErrorKind::SerializationFailure, + Box::new("restart transaction: Retry forever!".to_string()), + )) + }) + .await + .expect_err("Should have failed"); + + let samples = datastore + .transaction_retry_producer() + .samples + .lock() + .unwrap() + .clone(); + assert_eq!(samples.len(), MAX_RETRY_ATTEMPTS as usize); + + for i in 0..samples.len() { + let sample = &samples[i]; + + assert_eq!( + sample.timeseries_name, + "database_transaction:retry_data" + ); + + let target_fields = sample.sorted_target_fields(); + assert_eq!( + target_fields["name"].value, + FieldValue::String( + "test_transaction_retry_produces_samples".to_string() + ) + ); + + // Attempts are one-indexed + let metric_fields = sample.sorted_metric_fields(); + assert_eq!( + metric_fields["attempt"].value, + FieldValue::U32(u32::try_from(i).unwrap() + 1), + ); + } + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } +} diff --git a/nexus/src/app/background/dns_config.rs b/nexus/src/app/background/dns_config.rs index 654e9c0bf1..805ae813fe 100644 --- a/nexus/src/app/background/dns_config.rs +++ b/nexus/src/app/background/dns_config.rs @@ -166,7 +166,6 @@ mod test { use crate::app::background::init::test::read_internal_dns_zone_id; use crate::app::background::init::test::write_test_dns_generation; use assert_matches::assert_matches; - use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use async_bb8_diesel::AsyncSimpleConnection; use diesel::ExpressionMethods; @@ -237,11 +236,11 @@ mod test { ); // Similarly, wipe all of the state and verify that we handle that okay. + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper("dns_config_test_basic") + .transaction(&conn, |conn| async move { conn.batch_execute_async( nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL, ) @@ -265,7 +264,7 @@ mod test { .execute_async(&conn) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>(()) + Ok(()) }) .await .unwrap(); diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index cfa023a013..d30d2162c4 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -247,14 +247,12 @@ fn init_dns( #[cfg(test)] pub mod test { - use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use dropshot::HandlerTaskMode; use nexus_db_model::DnsGroup; use nexus_db_model::Generation; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; - use nexus_db_queries::db::TransactionError; use nexus_test_utils_macros::nexus_test; use nexus_types::internal_api::params as nexus_params; use nexus_types::internal_api::params::ServiceKind; @@ -446,11 +444,11 @@ pub mod test { datastore: &DataStore, internal_dns_zone_id: Uuid, ) { - type TxnError = TransactionError<()>; { let conn = datastore.pool_connection_for_tests().await.unwrap(); - let _: Result<(), TxnError> = conn - .transaction_async(|conn| async move { + let _: Result<(), _> = datastore + .transaction_retry_wrapper("write_test_dns_generation") + .transaction(&conn, |conn| async move { { use nexus_db_queries::db::model::DnsVersion; use nexus_db_queries::db::schema::dns_version::dsl; diff --git a/nexus/src/app/sagas/disk_create.rs b/nexus/src/app/sagas/disk_create.rs index fe403a7d41..4883afaddc 100644 --- a/nexus/src/app/sagas/disk_create.rs +++ b/nexus/src/app/sagas/disk_create.rs @@ -830,9 +830,7 @@ pub(crate) mod test { app::saga::create_saga_dag, app::sagas::disk_create::Params, app::sagas::disk_create::SagaDiskCreate, external_api::params, }; - use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use diesel::{ ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, }; @@ -972,27 +970,25 @@ pub(crate) mod test { use nexus_db_queries::db::model::VirtualProvisioningCollection; use nexus_db_queries::db::schema::virtual_provisioning_collection::dsl; + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper( + "no_virtual_provisioning_collection_records_using_storage", + ) + .transaction(&conn, |conn| async move { conn.batch_execute_async( nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL, ) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( - dsl::virtual_provisioning_collection - .filter(dsl::virtual_disk_bytes_provisioned.ne(0)) - .select(VirtualProvisioningCollection::as_select()) - .get_results_async::( - &conn, - ) - .await - .unwrap() - .is_empty(), - ) + Ok(dsl::virtual_provisioning_collection + .filter(dsl::virtual_disk_bytes_provisioned.ne(0)) + .select(VirtualProvisioningCollection::as_select()) + .get_results_async::(&conn) + .await + .unwrap() + .is_empty()) }) .await .unwrap() diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index 153e0323e7..8c2f96c36c 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -866,9 +866,7 @@ pub mod test { app::sagas::instance_create::SagaInstanceCreate, app::sagas::test_helpers, external_api::params, }; - use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use diesel::{ BoolExpressionMethods, ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, @@ -1013,30 +1011,28 @@ pub mod test { use nexus_db_queries::db::model::SledResource; use nexus_db_queries::db::schema::sled_resource::dsl; + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper( + "no_sled_resource_instance_records_exist", + ) + .transaction(&conn, |conn| async move { conn.batch_execute_async( nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL, ) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( - dsl::sled_resource - .filter( - dsl::kind.eq( - nexus_db_queries::db::model::SledResourceKind::Instance, - ), - ) - .select(SledResource::as_select()) - .get_results_async::(&conn) - .await - .unwrap() - .is_empty(), - ) + Ok(dsl::sled_resource + .filter(dsl::kind.eq( + nexus_db_queries::db::model::SledResourceKind::Instance, + )) + .select(SledResource::as_select()) + .get_results_async::(&conn) + .await + .unwrap() + .is_empty()) }) .await .unwrap() @@ -1048,16 +1044,17 @@ pub mod test { use nexus_db_queries::db::model::VirtualProvisioningResource; use nexus_db_queries::db::schema::virtual_provisioning_resource::dsl; - datastore.pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + let conn = datastore.pool_connection_for_tests().await.unwrap(); + + datastore + .transaction_retry_wrapper("no_virtual_provisioning_resource_records_exist") + .transaction(&conn, |conn| async move { conn .batch_execute_async(nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( + Ok( dsl::virtual_provisioning_resource .filter(dsl::resource_type.eq(nexus_db_queries::db::model::ResourceTypeProvisioned::Instance.to_string())) .select(VirtualProvisioningResource::as_select()) @@ -1075,31 +1072,29 @@ pub mod test { use nexus_db_queries::db::model::VirtualProvisioningCollection; use nexus_db_queries::db::schema::virtual_provisioning_collection::dsl; + let conn = datastore.pool_connection_for_tests().await.unwrap(); + datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper( + "no_virtual_provisioning_collection_records_using_instances", + ) + .transaction(&conn, |conn| async move { conn.batch_execute_async( nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL, ) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( - dsl::virtual_provisioning_collection - .filter( - dsl::cpus_provisioned - .ne(0) - .or(dsl::ram_provisioned.ne(0)), - ) - .select(VirtualProvisioningCollection::as_select()) - .get_results_async::( - &conn, - ) - .await - .unwrap() - .is_empty(), - ) + Ok(dsl::virtual_provisioning_collection + .filter( + dsl::cpus_provisioned + .ne(0) + .or(dsl::ram_provisioned.ne(0)), + ) + .select(VirtualProvisioningCollection::as_select()) + .get_results_async::(&conn) + .await + .unwrap() + .is_empty()) }) .await .unwrap() diff --git a/nexus/src/app/sagas/project_create.rs b/nexus/src/app/sagas/project_create.rs index 135e20ff06..40acc822c0 100644 --- a/nexus/src/app/sagas/project_create.rs +++ b/nexus/src/app/sagas/project_create.rs @@ -157,9 +157,7 @@ mod test { app::saga::create_saga_dag, app::sagas::project_create::Params, app::sagas::project_create::SagaProjectCreate, external_api::params, }; - use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, - }; + use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use diesel::{ ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, }; @@ -233,15 +231,16 @@ mod test { use nexus_db_queries::db::model::VirtualProvisioningCollection; use nexus_db_queries::db::schema::virtual_provisioning_collection::dsl; - datastore.pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + let conn = datastore.pool_connection_for_tests().await.unwrap(); + + datastore + .transaction_retry_wrapper("no_virtual_provisioning_collection_records_for_projects") + .transaction(&conn, |conn| async move { conn .batch_execute_async(nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL) .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( + Ok( dsl::virtual_provisioning_collection .filter(dsl::collection_type.eq(nexus_db_queries::db::model::CollectionTypeProvisioned::Project.to_string())) // ignore built-in services project diff --git a/nexus/src/app/sagas/test_helpers.rs b/nexus/src/app/sagas/test_helpers.rs index eccb013b66..3110bd318a 100644 --- a/nexus/src/app/sagas/test_helpers.rs +++ b/nexus/src/app/sagas/test_helpers.rs @@ -10,9 +10,7 @@ use crate::{ app::{saga::create_saga_dag, test_interfaces::TestInterfaces as _}, Nexus, }; -use async_bb8_diesel::{ - AsyncConnection, AsyncRunQueryDsl, AsyncSimpleConnection, -}; +use async_bb8_diesel::{AsyncRunQueryDsl, AsyncSimpleConnection}; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use futures::future::BoxFuture; use nexus_db_queries::{ @@ -434,11 +432,10 @@ pub(crate) async fn assert_no_failed_undo_steps( ) { use nexus_db_queries::db::model::saga_types::SagaNodeEvent; + let conn = datastore.pool_connection_for_tests().await.unwrap(); let saga_node_events: Vec = datastore - .pool_connection_for_tests() - .await - .unwrap() - .transaction_async(|conn| async move { + .transaction_retry_wrapper("assert_no_failed_undo_steps") + .transaction(&conn, |conn| async move { use nexus_db_queries::db::schema::saga_node_event::dsl; conn.batch_execute_async( @@ -447,14 +444,12 @@ pub(crate) async fn assert_no_failed_undo_steps( .await .unwrap(); - Ok::<_, nexus_db_queries::db::TransactionError<()>>( - dsl::saga_node_event - .filter(dsl::event_type.eq(String::from("undo_failed"))) - .select(SagaNodeEvent::as_select()) - .load_async::(&conn) - .await - .unwrap(), - ) + Ok(dsl::saga_node_event + .filter(dsl::event_type.eq(String::from("undo_failed"))) + .select(SagaNodeEvent::as_select()) + .load_async::(&conn) + .await + .unwrap()) }) .await .unwrap(); From 5185ab947941e79abc2446ee12e5111e698fb3a9 Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 05:17:50 +0000 Subject: [PATCH 15/16] Update taiki-e/install-action digest to d140130 (#4622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [taiki-e/install-action](https://togithub.com/taiki-e/install-action) | action | digest | [`d211c4b` -> `d140130`](https://togithub.com/taiki-e/install-action/compare/d211c4b...d140130) | --- ### Configuration 📅 **Schedule**: Branch creation - "after 8pm,before 6am" in timezone America/Los_Angeles, Automerge - "after 8pm,before 6am" in timezone America/Los_Angeles. 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://togithub.com/renovatebot/renovate). Co-authored-by: oxide-renovate[bot] <146848827+oxide-renovate[bot]@users.noreply.github.com> --- .github/workflows/hakari.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index 70b57a450a..0d1aec4c16 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -24,7 +24,7 @@ jobs: with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@d211c4be5a95cbcd52a0870dda7d63a107a58368 # v2 + uses: taiki-e/install-action@d140130aeedb5a946a5769684d32e3a33539f226 # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date From 001143cf3b686b8cb4a72916e9f775a0a186df9b Mon Sep 17 00:00:00 2001 From: "oxide-renovate[bot]" <146848827+oxide-renovate[bot]@users.noreply.github.com> Date: Tue, 5 Dec 2023 21:23:47 -0800 Subject: [PATCH 16/16] Update Rust crate filetime to 0.2.23 (#4623) --- Cargo.lock | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 2 +- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13b3d6c74e..981dd99082 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2173,14 +2173,14 @@ checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" [[package]] name = "filetime" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", + "redox_syscall 0.4.1", + "windows-sys 0.52.0", ] [[package]] @@ -9652,6 +9652,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -9682,6 +9691,21 @@ dependencies = [ "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -9694,6 +9718,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -9706,6 +9736,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -9718,6 +9754,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -9730,6 +9772,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -9742,6 +9790,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -9754,6 +9808,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -9766,6 +9826,12 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winnow" version = "0.5.15" diff --git a/Cargo.toml b/Cargo.toml index c88502bb1c..3a80367806 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -191,7 +191,7 @@ dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", either = "1.9.0" expectorate = "1.1.0" fatfs = "0.3.6" -filetime = "0.2.22" +filetime = "0.2.23" flate2 = "1.0.28" flume = "0.11.0" foreign-types = "0.3.2"