From 617990503cbd9cf4968327e14d45da0251563ec7 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Tue, 2 Jul 2024 13:58:09 -0700 Subject: [PATCH] Reconfigurator: Decommission cockroach nodes that belong to expunged cockroach omicron zones (#5903) This adds a "decommission a node" endpoint to the `cockroach-admin` server, and a step to blueprint execution to clean up any expunged zones (which for now, only does anything for cockroach). --- Cargo.lock | 3 + cockroach-admin/Cargo.toml | 2 + .../proptest-regressions/cockroach_cli.txt | 7 + cockroach-admin/src/cockroach_cli.rs | 340 ++++++++++++++- cockroach-admin/src/http_entrypoints.rs | 35 +- dev-tools/reconfigurator-cli/src/main.rs | 14 + nexus/reconfigurator/execution/Cargo.toml | 1 + .../execution/src/cockroachdb.rs | 2 + nexus/reconfigurator/execution/src/dns.rs | 10 + nexus/reconfigurator/execution/src/lib.rs | 12 + .../execution/src/omicron_zones.rs | 400 +++++++++++++++++- nexus/src/app/background/init.rs | 1 + .../background/tasks/blueprint_execution.rs | 8 +- .../tasks/crdb_node_id_collector.rs | 6 +- nexus/src/app/instance.rs | 2 +- nexus/src/app/instance_network.rs | 9 +- nexus/src/app/mod.rs | 12 +- nexus/src/app/sagas/common_storage.rs | 1 - nexus/types/src/deployment.rs | 6 + openapi/cockroach-admin.json | 154 ++++++- 20 files changed, 968 insertions(+), 57 deletions(-) create mode 100644 cockroach-admin/proptest-regressions/cockroach_cli.txt diff --git a/Cargo.lock b/Cargo.lock index 5a77a6bbdc..ccf6137f1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4806,6 +4806,7 @@ dependencies = [ "anyhow", "async-bb8-diesel", "chrono", + "cockroach-admin-client", "diesel", "dns-service-client", "futures", @@ -5300,6 +5301,7 @@ dependencies = [ "openapi-lint", "openapiv3", "pq-sys", + "proptest", "schemars", "serde", "serde_json", @@ -5308,6 +5310,7 @@ dependencies = [ "slog-dtrace", "slog-error-chain", "subprocess", + "test-strategy", "thiserror", "tokio", "tokio-postgres", diff --git a/cockroach-admin/Cargo.toml b/cockroach-admin/Cargo.toml index 49401afb9d..07f9807463 100644 --- a/cockroach-admin/Cargo.toml +++ b/cockroach-admin/Cargo.toml @@ -40,8 +40,10 @@ nexus-test-utils.workspace = true omicron-test-utils.workspace = true openapi-lint.workspace = true openapiv3.workspace = true +proptest.workspace = true serde_json.workspace = true subprocess.workspace = true +test-strategy.workspace = true url.workspace = true [lints] diff --git a/cockroach-admin/proptest-regressions/cockroach_cli.txt b/cockroach-admin/proptest-regressions/cockroach_cli.txt new file mode 100644 index 0000000000..7583f353c0 --- /dev/null +++ b/cockroach-admin/proptest-regressions/cockroach_cli.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 924830db4b84d94683d81ff27010fec0df19a72155729e60fda503b58460390e # shrinks to input = _NodeDecommissionParseDoesntPanicOnArbitraryInputArgs { input: [10, 10] } diff --git a/cockroach-admin/src/cockroach_cli.rs b/cockroach-admin/src/cockroach_cli.rs index 00478b81a1..1951866ce7 100644 --- a/cockroach-admin/src/cockroach_cli.rs +++ b/cockroach-admin/src/cockroach_cli.rs @@ -82,10 +82,41 @@ impl CockroachCli { pub async fn node_status( &self, ) -> Result, CockroachCliError> { + self.invoke_cli_with_format_csv( + ["node", "status"].into_iter(), + NodeStatus::parse_from_csv, + "node status", + ) + .await + } + + pub async fn node_decommission( + &self, + node_id: &str, + ) -> Result { + self.invoke_cli_with_format_csv( + ["node", "decommission", node_id, "--wait", "none"].into_iter(), + NodeDecommission::parse_from_csv, + "node decommission", + ) + .await + } + + async fn invoke_cli_with_format_csv<'a, F, I, T>( + &self, + subcommand_args: I, + parse_output: F, + subcommand_description: &'static str, + ) -> Result + where + F: FnOnce(&[u8]) -> Result, + I: Iterator, + { let mut command = Command::new(&self.path_to_cockroach_binary); + for arg in subcommand_args { + command.arg(arg); + } command - .arg("node") - .arg("status") .arg("--host") .arg(&format!("{}", self.cockroach_address)) .arg("--insecure") @@ -97,14 +128,14 @@ impl CockroachCli { if !output.status.success() { return Err(output_to_exec_error(command.as_std(), &output).into()); } - NodeStatus::parse_from_csv(io::Cursor::new(&output.stdout)).map_err( - |err| CockroachCliError::ParseOutput { - subcommand: "node status", + parse_output(&output.stdout).map_err(|err| { + CockroachCliError::ParseOutput { + subcommand: subcommand_description, stdout: String::from_utf8_lossy(&output.stdout).to_string(), stderr: String::from_utf8_lossy(&output.stderr).to_string(), err, - }, - ) + } + }) } } @@ -123,10 +154,8 @@ pub struct NodeStatus { } // Slightly different `NodeStatus` that matches what we get from `cockroach`: -// -// * `id` column instead of `node_id` -// * timestamps are a fixed format with no timezone, so we have a custom -// deserializer +// timestamps are a fixed format with no timezone (but are actually UTC), so we +// have a custom deserializer, and the ID column is `id` instead of `node_id`. #[derive(Debug, Clone, PartialEq, Eq, Deserialize)] struct CliNodeStatus { id: String, @@ -189,12 +218,9 @@ where } impl NodeStatus { - pub fn parse_from_csv(reader: R) -> Result, csv::Error> - where - R: io::Read, - { + pub fn parse_from_csv(data: &[u8]) -> Result, csv::Error> { let mut statuses = Vec::new(); - let mut reader = csv::Reader::from_reader(reader); + let mut reader = csv::Reader::from_reader(io::Cursor::new(data)); for result in reader.deserialize() { let record: CliNodeStatus = result?; statuses.push(record.into()); @@ -203,17 +229,146 @@ impl NodeStatus { } } +// The cockroach CLI and `crdb_internal.gossip_liveness` table use a string for +// node membership, but there are only three meaningful values per +// https://github.com/cockroachdb/cockroach/blob/0c92c710d2baadfdc5475be8d2238cf26cb152ca/pkg/kv/kvserver/liveness/livenesspb/liveness.go#L96, +// so we'll convert into a Rust enum and leave the "unknown" case for future +// changes that expand or reword these values. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(tag = "state", rename_all = "lowercase")] +pub enum NodeMembership { + Active, + Decommissioning, + Decommissioned, + Unknown { value: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct NodeDecommission { + pub node_id: String, + pub is_live: bool, + pub replicas: i64, + pub is_decommissioning: bool, + pub membership: NodeMembership, + pub is_draining: bool, + pub notes: Vec, +} + +// Slightly different `NodeDecommission` that matches what we get from +// `cockroach`: this omites `notes`, which isn't really a CSV field at all, but +// is instead where we collect the non-CSV string output from the CLI, uses +// a custom deserializer for `membership` to handle unknown variants, and the ID +// column is `id` instead of `node_id`. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +struct CliNodeDecommission { + pub id: String, + pub is_live: bool, + pub replicas: i64, + pub is_decommissioning: bool, + #[serde(deserialize_with = "parse_node_membership")] + pub membership: NodeMembership, + pub is_draining: bool, +} + +impl From<(CliNodeDecommission, Vec)> for NodeDecommission { + fn from((cli, notes): (CliNodeDecommission, Vec)) -> Self { + Self { + node_id: cli.id, + is_live: cli.is_live, + replicas: cli.replicas, + is_decommissioning: cli.is_decommissioning, + membership: cli.membership, + is_draining: cli.is_draining, + notes, + } + } +} + +fn parse_node_membership<'de, D>(d: D) -> Result +where + D: serde::Deserializer<'de>, +{ + struct CockroachNodeMembershipVisitor; + + impl<'de> de::Visitor<'de> for CockroachNodeMembershipVisitor { + type Value = NodeMembership; + + fn expecting( + &self, + formatter: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + formatter.write_str("a Cockroach node membership string") + } + + fn visit_str(self, v: &str) -> Result + where + E: de::Error, + { + let membership = match v { + "active" => NodeMembership::Active, + "decommissioning" => NodeMembership::Decommissioning, + "decommissioned" => NodeMembership::Decommissioned, + _ => NodeMembership::Unknown { value: v.to_string() }, + }; + Ok(membership) + } + } + + d.deserialize_str(CockroachNodeMembershipVisitor) +} + +impl NodeDecommission { + pub fn parse_from_csv(data: &[u8]) -> Result { + // Reading the node decommission output is awkward because it isn't + // fully CSV. We expect a CSV header, then a row for each node being + // decommissioned, then (maybe) a blank line followed by a note that is + // just a string, not related to the initial CSV data. Even though the + // CLI supports decommissioning more than one node in one invocation, we + // only provide an API to decommission a single node, so we expect: + // + // 1. The CSV header line + // 2. The one row of CSV data + // 3. Trailing notes + // + // We'll collect the notes as a separate field and return them to our + // caller. + + // First we'll run the data through a csv::Reader; this will pull out + // the header row and the one row of data. + let mut reader = csv::Reader::from_reader(io::Cursor::new(data)); + let record: CliNodeDecommission = + reader.deserialize().next().ok_or_else(|| { + io::Error::other("fewer than two lines of output") + })??; + + // Get the position where the reader ended after that one row; we'll + // collect any remaining nonempty lines as `notes`. + let extra_data = &data[reader.position().byte() as usize..]; + let mut notes = Vec::new(); + for line in String::from_utf8_lossy(extra_data).lines() { + let line = line.trim(); + if !line.is_empty() { + notes.push(line.to_string()); + } + } + + Ok(Self::from((record, notes))) + } +} + #[cfg(test)] mod tests { use super::*; use chrono::NaiveDate; use nexus_test_utils::db::test_setup_database; use omicron_test_utils::dev; + use test_strategy::proptest; use url::Url; #[test] fn test_node_status_parse_single_line_from_csv() { - let input = r#"id,address,sql_address,build,started_at,updated_at,locality,is_available,is_live + let input = br#"id,address,sql_address,build,started_at,updated_at,locality,is_available,is_live 1,[::1]:42021,[::1]:42021,v22.1.9,2024-05-21 15:19:50.523796,2024-05-21 16:31:28.050069,,true,true"#; let expected = NodeStatus { node_id: "1".to_string(), @@ -239,14 +394,13 @@ mod tests { is_live: true, }; - let statuses = NodeStatus::parse_from_csv(io::Cursor::new(input)) - .expect("parsed input"); + let statuses = NodeStatus::parse_from_csv(input).expect("parsed input"); assert_eq!(statuses, vec![expected]); } #[test] fn test_node_status_parse_multiple_lines_from_csv() { - let input = r#"id,address,sql_address,build,started_at,updated_at,locality,is_available,is_live + let input = br#"id,address,sql_address,build,started_at,updated_at,locality,is_available,is_live 1,[fd00:1122:3344:109::3]:32221,[fd00:1122:3344:109::3]:32221,v22.1.9-dirty,2024-05-18 19:18:00.597145,2024-05-21 15:22:34.290434,,true,true 2,[fd00:1122:3344:105::3]:32221,[fd00:1122:3344:105::3]:32221,v22.1.9-dirty,2024-05-18 19:17:01.796714,2024-05-21 15:22:34.901268,,true,true 3,[fd00:1122:3344:10b::3]:32221,[fd00:1122:3344:10b::3]:32221,v22.1.9-dirty,2024-05-18 19:18:52.37564,2024-05-21 15:22:36.341146,,true,true @@ -370,14 +524,78 @@ mod tests { }, ]; - let statuses = NodeStatus::parse_from_csv(io::Cursor::new(input)) - .expect("parsed input"); + let statuses = NodeStatus::parse_from_csv(input).expect("parsed input"); assert_eq!(statuses.len(), expected.len()); for (status, expected) in statuses.iter().zip(&expected) { assert_eq!(status, expected); } } + #[test] + fn test_node_decommission_parse_with_no_trailing_notes() { + let input = + br#"id,is_live,replicas,is_decommissioning,membership,is_draining +6,true,24,true,decommissioning,false"#; + let expected = NodeDecommission { + node_id: "6".to_string(), + is_live: true, + replicas: 24, + is_decommissioning: true, + membership: NodeMembership::Decommissioning, + is_draining: false, + notes: vec![], + }; + + let statuses = + NodeDecommission::parse_from_csv(input).expect("parsed input"); + assert_eq!(statuses, expected); + } + + #[test] + fn test_node_decommission_parse_with_trailing_notes() { + let input = + br#"id,is_live,replicas,is_decommissioning,membership,is_draining +6,false,0,true,decommissioned,false + +No more data reported on target nodes. Please verify cluster health before removing the nodes. +"#; + let expected = NodeDecommission { + node_id: "6".to_string(), + is_live: false, + replicas: 0, + is_decommissioning: true, + membership: NodeMembership::Decommissioned, + is_draining: false, + notes: vec!["No more data reported on target nodes. \ + Please verify cluster health before removing the nodes." + .to_string()], + }; + + let statuses = + NodeDecommission::parse_from_csv(input).expect("parsed input"); + assert_eq!(statuses, expected); + } + + #[test] + fn test_node_decommission_parse_with_unexpected_membership_value() { + let input = + br#"id,is_live,replicas,is_decommissioning,membership,is_draining +6,false,0,true,foobar,false"#; + let expected = NodeDecommission { + node_id: "6".to_string(), + is_live: false, + replicas: 0, + is_decommissioning: true, + membership: NodeMembership::Unknown { value: "foobar".to_string() }, + is_draining: false, + notes: vec![], + }; + + let statuses = + NodeDecommission::parse_from_csv(input).expect("parsed input"); + assert_eq!(statuses, expected); + } + // Ensure that if `cockroach node status` changes in a future CRDB version // bump, we have a test that will fail to force us to check whether our // current parsing is still valid. @@ -435,4 +653,82 @@ mod tests { db.cleanup().await.unwrap(); logctx.cleanup_successful(); } + + // Ensure that if `cockroach node decommission` changes in a future CRDB + // version bump, we have a test that will fail to force us to check whether + // our current parsing is still valid. + #[tokio::test] + async fn test_node_decommission_compatibility() { + let logctx = + dev::test_setup_log("test_node_decommission_compatibility"); + let mut db = test_setup_database(&logctx.log).await; + let db_url = db.listen_url().to_string(); + + let expected_headers = + "id,is_live,replicas,is_decommissioning,membership,is_draining"; + + // Manually run cockroach node decommission to grab just the CSV header + // line (which the `csv` crate normally eats on our behalf) and check + // it's exactly what we expect. + let mut command = Command::new("cockroach"); + command + .arg("node") + .arg("decommission") + .arg("1") + .arg("--wait") + .arg("none") + .arg("--url") + .arg(&db_url) + .arg("--format") + .arg("csv"); + let output = + command.output().await.expect("ran `cockroach node decommission`"); + + let stdout = String::from_utf8_lossy(&output.stdout); + let mut lines = stdout.lines(); + let headers = lines.next().expect("header line"); + assert_eq!( + headers, expected_headers, + "`cockroach node decommission --format csv` headers \ + may have changed?" + ); + + // We should also be able to run our wrapper against this cockroach. + let url: Url = db_url.parse().expect("valid url"); + let cockroach_address: SocketAddrV6 = format!( + "{}:{}", + url.host().expect("url has host"), + url.port().expect("url has port") + ) + .parse() + .expect("valid SocketAddrV6"); + let cli = CockroachCli::new("cockroach".into(), cockroach_address); + let result = cli + .node_decommission("1") + .await + .expect("got node decommission result"); + + // We can't check all the fields exactly (e.g., replicas), but most we + // know based on the fact that our test database is a single node, so + // won't actually decommission itself. + assert_eq!(result.node_id, "1"); + assert_eq!(result.is_live, true); + assert_eq!(result.is_decommissioning, true); + assert_eq!(result.membership, NodeMembership::Decommissioning); + assert_eq!(result.is_draining, false); + assert_eq!(result.notes, &[] as &[&str]); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + + #[proptest] + fn node_status_parse_doesnt_panic_on_arbitrary_input(input: Vec) { + _ = NodeStatus::parse_from_csv(&input); + } + + #[proptest] + fn node_decommission_parse_doesnt_panic_on_arbitrary_input(input: Vec) { + _ = NodeDecommission::parse_from_csv(&input); + } } diff --git a/cockroach-admin/src/http_entrypoints.rs b/cockroach-admin/src/http_entrypoints.rs index bf12eb933b..1c02d23ae2 100644 --- a/cockroach-admin/src/http_entrypoints.rs +++ b/cockroach-admin/src/http_entrypoints.rs @@ -2,12 +2,14 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::cockroach_cli::NodeDecommission; use crate::cockroach_cli::NodeStatus; use crate::context::ServerContext; use dropshot::endpoint; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::RequestContext; +use dropshot::TypedBody; use omicron_uuid_kinds::OmicronZoneUuid; use schemars::JsonSchema; use serde::Deserialize; @@ -18,8 +20,9 @@ type CrdbApiDescription = dropshot::ApiDescription>; pub fn api() -> CrdbApiDescription { fn register_endpoints(api: &mut CrdbApiDescription) -> Result<(), String> { - api.register(node_id)?; + api.register(local_node_id)?; api.register(node_status)?; + api.register(node_decommission)?; Ok(()) } @@ -53,7 +56,7 @@ async fn node_status( /// CockroachDB Node ID #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "snake_case")] -pub struct NodeId { +pub struct LocalNodeId { /// The ID of this Omicron zone. /// /// This is included to ensure correctness even if a socket address on a @@ -75,11 +78,33 @@ pub struct NodeId { method = GET, path = "/node/id", }] -async fn node_id( +async fn local_node_id( rqctx: RequestContext>, -) -> Result, HttpError> { +) -> Result, HttpError> { let ctx = rqctx.context(); let node_id = ctx.node_id().await?.to_string(); let zone_id = ctx.zone_id(); - Ok(HttpResponseOk(NodeId { zone_id, node_id })) + Ok(HttpResponseOk(LocalNodeId { zone_id, node_id })) +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct NodeId { + pub node_id: String, +} + +/// Decommission a node from the CRDB cluster +#[endpoint { + method = POST, + path = "/node/decommission", +}] +async fn node_decommission( + rqctx: RequestContext>, + body: TypedBody, +) -> Result, HttpError> { + let ctx = rqctx.context(); + let NodeId { node_id } = body.into_inner(); + let decommission_status = + ctx.cockroach_cli().node_decommission(&node_id).await?; + Ok(HttpResponseOk(decommission_status)) } diff --git a/dev-tools/reconfigurator-cli/src/main.rs b/dev-tools/reconfigurator-cli/src/main.rs index bc212281b2..3234a3fdc2 100644 --- a/dev-tools/reconfigurator-cli/src/main.rs +++ b/dev-tools/reconfigurator-cli/src/main.rs @@ -433,6 +433,8 @@ enum BlueprintEditCommands { /// sled on which to deploy the new instance sled_id: SledUuid, }, + /// add a CockroachDB instance to a particular sled + AddCockroach { sled_id: SledUuid }, } #[derive(Debug, Args)] @@ -756,6 +758,18 @@ fn cmd_blueprint_edit( ); format!("added Nexus zone to sled {}", sled_id) } + BlueprintEditCommands::AddCockroach { sled_id } => { + let current = + builder.sled_num_zones_of_kind(sled_id, ZoneKind::CockroachDb); + let added = builder + .sled_ensure_zone_multiple_cockroachdb(sled_id, current + 1) + .context("failed to add CockroachDB zone")?; + assert_matches::assert_matches!( + added, + EnsureMultiple::Changed { added: 1, removed: 0 } + ); + format!("added CockroachDB zone to sled {}", sled_id) + } }; let new_blueprint = builder.build(); diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml index 34056b45a1..00103528bb 100644 --- a/nexus/reconfigurator/execution/Cargo.toml +++ b/nexus/reconfigurator/execution/Cargo.toml @@ -11,6 +11,7 @@ omicron-rpaths.workspace = true [dependencies] anyhow.workspace = true +cockroach-admin-client.workspace = true dns-service-client.workspace = true chrono.workspace = true futures.workspace = true diff --git a/nexus/reconfigurator/execution/src/cockroachdb.rs b/nexus/reconfigurator/execution/src/cockroachdb.rs index 101a7372c5..6bd72955c7 100644 --- a/nexus/reconfigurator/execution/src/cockroachdb.rs +++ b/nexus/reconfigurator/execution/src/cockroachdb.rs @@ -49,6 +49,7 @@ mod test { ) { let nexus = &cptestctx.server.server_context().nexus; let datastore = nexus.datastore(); + let resolver = nexus.resolver(); let log = &cptestctx.logctx.log; let opctx = OpContext::for_background( log.clone(), @@ -92,6 +93,7 @@ mod test { crate::realize_blueprint_with_overrides( &opctx, datastore, + resolver, &blueprint, "test-suite", &overrides, diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index c5e954ef2c..f3b718ee54 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -457,6 +457,7 @@ mod test { use crate::overridables::Overridables; use crate::Sled; use dns_service_client::DnsDiff; + use internal_dns::resolver::Resolver; use internal_dns::ServiceName; use internal_dns::DNS_ZONE; use nexus_db_model::DnsGroup; @@ -1150,6 +1151,7 @@ mod test { let nexus = &cptestctx.server.server_context().nexus; let datastore = nexus.datastore(); + let resolver = nexus.resolver(); let log = &cptestctx.logctx.log; let opctx = OpContext::for_background( log.clone(), @@ -1183,6 +1185,7 @@ mod test { crate::realize_blueprint_with_overrides( &opctx, datastore, + resolver, &blueprint, "test-suite", &overrides, @@ -1207,6 +1210,7 @@ mod test { cptestctx, &opctx, datastore, + resolver, &blueprint, &overrides, "squidport", @@ -1319,6 +1323,7 @@ mod test { crate::realize_blueprint_with_overrides( &opctx, datastore, + resolver, &blueprint2, "test-suite", &overrides, @@ -1392,6 +1397,7 @@ mod test { crate::realize_blueprint_with_overrides( &opctx, datastore, + resolver, &blueprint2, "test-suite", &overrides, @@ -1413,6 +1419,7 @@ mod test { &cptestctx, &opctx, datastore, + resolver, &blueprint2, &overrides, "tickety-boo", @@ -1426,6 +1433,7 @@ mod test { crate::realize_blueprint_with_overrides( &opctx, datastore, + resolver, &blueprint2, "test-suite", &overrides, @@ -1471,6 +1479,7 @@ mod test { cptestctx: &ControlPlaneTestContext, opctx: &OpContext, datastore: &DataStore, + resolver: &Resolver, blueprint: &Blueprint, overrides: &Overridables, silo_name: &str, @@ -1518,6 +1527,7 @@ mod test { crate::realize_blueprint_with_overrides( &opctx, datastore, + resolver, &blueprint, "test-suite", &overrides, diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs index 63bb4b24f0..0e9ab394f1 100644 --- a/nexus/reconfigurator/execution/src/lib.rs +++ b/nexus/reconfigurator/execution/src/lib.rs @@ -7,6 +7,7 @@ //! See `nexus_reconfigurator_planning` crate-level docs for background. use anyhow::{anyhow, Context}; +use internal_dns::resolver::Resolver; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::Blueprint; @@ -76,6 +77,7 @@ impl From for Sled { pub async fn realize_blueprint( opctx: &OpContext, datastore: &DataStore, + resolver: &Resolver, blueprint: &Blueprint, nexus_label: S, ) -> Result<(), Vec> @@ -85,6 +87,7 @@ where realize_blueprint_with_overrides( opctx, datastore, + resolver, blueprint, nexus_label, &Default::default(), @@ -95,6 +98,7 @@ where pub async fn realize_blueprint_with_overrides( opctx: &OpContext, datastore: &DataStore, + resolver: &Resolver, blueprint: &Blueprint, nexus_label: S, overrides: &Overridables, @@ -204,6 +208,14 @@ where .await .map_err(|e| vec![anyhow!("{}", InlineErrorChain::new(&e))])?; + omicron_zones::clean_up_expunged_zones( + &opctx, + datastore, + resolver, + blueprint.all_omicron_zones(BlueprintZoneFilter::Expunged), + ) + .await?; + sled_state::decommission_sleds( &opctx, datastore, diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs index 08d41928c9..404124ba25 100644 --- a/nexus/reconfigurator/execution/src/omicron_zones.rs +++ b/nexus/reconfigurator/execution/src/omicron_zones.rs @@ -6,17 +6,32 @@ use crate::Sled; use anyhow::anyhow; +use anyhow::bail; use anyhow::Context; +use cockroach_admin_client::types::NodeDecommission; +use cockroach_admin_client::types::NodeId; use futures::stream; use futures::StreamExt; +use internal_dns::resolver::Resolver; +use internal_dns::ServiceName; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::DataStore; +use nexus_types::deployment::BlueprintZoneConfig; +use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneFilter; +use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::BlueprintZonesConfig; +use omicron_common::address::COCKROACH_ADMIN_PORT; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; use slog::info; use slog::warn; +use slog::Logger; +use slog_error_chain::InlineErrorChain; use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::net::SocketAddrV6; /// Idempotently ensure that the specified Omicron zones are deployed to the /// corresponding sleds @@ -85,22 +100,219 @@ pub(crate) async fn deploy_zones( } } +/// Idempontently perform any cleanup actions necessary for expunged zones. +pub(crate) async fn clean_up_expunged_zones( + opctx: &OpContext, + datastore: &DataStore, + resolver: &R, + expunged_zones: impl Iterator, +) -> Result<(), Vec> { + let errors: Vec = stream::iter(expunged_zones) + .filter_map(|(sled_id, config)| async move { + // We expect to only be called with expunged zones; skip any with a + // different disposition. + if config.disposition != BlueprintZoneDisposition::Expunged { + return None; + } + + let log = opctx.log.new(slog::o!( + "sled_id" => sled_id.to_string(), + "zone_id" => config.id.to_string(), + "zone_type" => config.zone_type.kind().to_string(), + )); + + let result = match &config.zone_type { + // Zones which need no cleanup work after expungement. + BlueprintZoneType::Nexus(_) => None, + + // Zones which need cleanup after expungement. + BlueprintZoneType::CockroachDb(_) => Some( + decommission_cockroachdb_node( + opctx, datastore, resolver, config.id, &log, + ) + .await, + ), + + // Zones that may or may not need cleanup work - we haven't + // gotten to these yet! + BlueprintZoneType::BoundaryNtp(_) + | BlueprintZoneType::Clickhouse(_) + | BlueprintZoneType::ClickhouseKeeper(_) + | BlueprintZoneType::Crucible(_) + | BlueprintZoneType::CruciblePantry(_) + | BlueprintZoneType::ExternalDns(_) + | BlueprintZoneType::InternalDns(_) + | BlueprintZoneType::InternalNtp(_) + | BlueprintZoneType::Oximeter(_) => { + warn!( + log, + "unsupported zone type for expungement cleanup; \ + not performing any cleanup"; + ); + None + } + }?; + + match result { + Err(error) => { + warn!( + log, "failed to clean up expunged zone"; + "error" => #%error, + ); + Some(error) + } + Ok(()) => { + info!(log, "successfully cleaned up after expunged zone"); + None + } + } + }) + .collect() + .await; + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } +} + +// Helper trait that is implemented by `Resolver`, but allows unit tests to +// inject a fake resolver that points to a mock server when calling +// `decommission_cockroachdb_node()`. +pub(crate) trait CleanupResolver { + async fn resolve_cockroach_admin_addresses( + &self, + ) -> anyhow::Result>; +} + +impl CleanupResolver for Resolver { + async fn resolve_cockroach_admin_addresses( + &self, + ) -> anyhow::Result> { + let crdb_ips = self + .lookup_all_ipv6(ServiceName::Cockroach) + .await + .context("failed to resolve cockroach IPs in internal DNS")?; + let ip_to_admin_addr = |ip| { + SocketAddr::V6(SocketAddrV6::new(ip, COCKROACH_ADMIN_PORT, 0, 0)) + }; + Ok(crdb_ips.into_iter().map(ip_to_admin_addr)) + } +} + +async fn decommission_cockroachdb_node( + opctx: &OpContext, + datastore: &DataStore, + resolver: &R, + zone_id: OmicronZoneUuid, + log: &Logger, +) -> anyhow::Result<()> { + // We need the node ID of this zone. Check and see whether the + // crdb_node_id_collector RPW found the node ID for this zone. If it hasn't, + // we're in trouble: we don't know whether this zone came up far enough to + // join the cockroach cluster (and therefore needs decommissioning) but was + // expunged before the collector RPW could identify it, or if the zone never + // joined the cockroach cluster (and therefore doesn't have a node ID and + // doesn't need decommissioning). + // + // For now, we punt on this problem. If we were certain that the zone's + // socket address could never be reused, we could ask one of the running + // cockroach nodes for the status of all nodes, find the ID of the one with + // this zone's listening address (if one exists), and decommission that ID. + // But if the address could be reused, that risks decommissioning a live + // node! We'll just log a warning and require a human to figure out how to + // clean this up. + // + // TODO-cleanup Can we decommission nodes in this state automatically? If + // not, can we be noisier than just a `warn!` log without failing blueprint + // exeuction entirely? + let Some(node_id) = + datastore.cockroachdb_node_id(opctx, zone_id).await.with_context( + || format!("failed to look up node ID of cockroach zone {zone_id}"), + )? + else { + warn!( + log, + "expunged cockroach zone has no known node ID; \ + support intervention is required for zone cleanup" + ); + return Ok(()); + }; + + // To decommission a CRDB node, we need to talk to one of the still-running + // nodes; look them up in DNS try them all in order. (It would probably be + // fine to try them all concurrently, but this isn't a very common + // operation, so keeping it simple seems fine.) + // + // TODO-cleanup: Replace this with a qorb-based connection pool once it's + // ready. + let admin_addrs = resolver.resolve_cockroach_admin_addresses().await?; + + let mut num_admin_addrs_tried = 0; + for admin_addr in admin_addrs { + let admin_url = format!("http://{admin_addr}"); + let log = log.new(slog::o!("admin_url" => admin_url.clone())); + let client = + cockroach_admin_client::Client::new(&admin_url, log.clone()); + + let body = NodeId { node_id: node_id.clone() }; + match client + .node_decommission(&body) + .await + .map(|response| response.into_inner()) + { + Ok(NodeDecommission { + is_decommissioning, + is_draining, + is_live, + membership, + node_id: _, + notes, + replicas, + }) => { + info!( + log, "successfully sent cockroach decommission request"; + "is_decommissioning" => is_decommissioning, + "is_draining" => is_draining, + "is_live" => is_live, + "membership" => ?membership, + "notes" => ?notes, + "replicas" => replicas, + ); + + return Ok(()); + } + + Err(err) => { + warn!( + log, "failed sending decommission request \ + (will try other servers)"; + "err" => InlineErrorChain::new(&err), + ); + num_admin_addrs_tried += 1; + } + } + } + + bail!( + "failed to decommission cockroach zone {zone_id} (node {node_id}): \ + failed to contact {num_admin_addrs_tried} admin servers" + ); +} + #[cfg(test)] mod test { - use super::deploy_zones; + use super::*; use crate::Sled; + use cockroach_admin_client::types::NodeMembership; use httptest::matchers::{all_of, json_decoded, request}; - use httptest::responders::status_code; + use httptest::responders::{json_encoded, status_code}; use httptest::Expectation; - use nexus_db_queries::context::OpContext; use nexus_test_utils_macros::nexus_test; use nexus_types::deployment::{ - blueprint_zone_type, BlueprintZoneType, CockroachDbPreserveDowngrade, - OmicronZonesConfig, - }; - use nexus_types::deployment::{ - Blueprint, BlueprintTarget, BlueprintZoneConfig, - BlueprintZoneDisposition, BlueprintZonesConfig, + blueprint_zone_type, Blueprint, BlueprintTarget, + CockroachDbPreserveDowngrade, OmicronZonesConfig, }; use nexus_types::inventory::OmicronZoneDataset; use omicron_common::api::external::Generation; @@ -109,7 +321,7 @@ mod test { use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use std::collections::BTreeMap; - use std::net::SocketAddr; + use std::iter; use uuid::Uuid; type ControlPlaneTestContext = @@ -346,4 +558,172 @@ mod test { s1.verify_and_clear(); s2.verify_and_clear(); } + + #[nexus_test] + async fn test_clean_up_cockroach_zones( + cptestctx: &ControlPlaneTestContext, + ) { + // Test setup boilerplate. + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + + // Construct the cockroach zone we're going to try to clean up. + let any_sled_id = SledUuid::new_v4(); + let crdb_zone = BlueprintZoneConfig { + disposition: BlueprintZoneDisposition::Expunged, + id: OmicronZoneUuid::new_v4(), + underlay_address: "::1".parse().unwrap(), + filesystem_pool: Some(ZpoolName::new_external(ZpoolUuid::new_v4())), + zone_type: BlueprintZoneType::CockroachDb( + blueprint_zone_type::CockroachDb { + address: "[::1]:0".parse().unwrap(), + dataset: OmicronZoneDataset { + pool_name: format!("oxp_{}", Uuid::new_v4()) + .parse() + .unwrap(), + }, + }, + ), + }; + + // Start a mock cockroach-admin server. + let mut mock_admin = httptest::Server::run(); + + // Create our fake resolver that will point to our mock server. + struct FixedResolver(Vec); + impl CleanupResolver for FixedResolver { + async fn resolve_cockroach_admin_addresses( + &self, + ) -> anyhow::Result> { + Ok(self.0.clone().into_iter()) + } + } + let fake_resolver = FixedResolver(vec![mock_admin.addr()]); + + // We haven't yet inserted a mapping from zone ID to cockroach node ID + // in the db, so trying to clean up the zone should log a warning but + // otherwise succeed, without attempting to contact our mock admin + // server. (We don't have a good way to confirm the warning was logged, + // so we'll just check for an Ok return and no contact to mock_admin.) + clean_up_expunged_zones( + &opctx, + datastore, + &fake_resolver, + iter::once((any_sled_id, &crdb_zone)), + ) + .await + .expect("unknown node ID: no cleanup"); + mock_admin.verify_and_clear(); + + // Record a zone ID <-> node ID mapping. + let crdb_node_id = "test-node"; + datastore + .set_cockroachdb_node_id( + &opctx, + crdb_zone.id, + crdb_node_id.to_string(), + ) + .await + .expect("assigned node ID"); + + // Cleaning up the zone should now contact the mock-admin server and + // attempt to decommission it. + let add_decommission_expecation = + move |mock_server: &mut httptest::Server| { + mock_server.expect( + Expectation::matching(all_of![ + request::method_path("POST", "/node/decommission"), + request::body(json_decoded(move |n: &NodeId| { + n.node_id == crdb_node_id + })) + ]) + .respond_with(json_encoded( + NodeDecommission { + is_decommissioning: true, + is_draining: true, + is_live: false, + membership: NodeMembership::Decommissioning, + node_id: crdb_node_id.to_string(), + notes: vec![], + replicas: 0, + }, + )), + ); + }; + add_decommission_expecation(&mut mock_admin); + clean_up_expunged_zones( + &opctx, + datastore, + &fake_resolver, + iter::once((any_sled_id, &crdb_zone)), + ) + .await + .expect("decommissioned test node"); + mock_admin.verify_and_clear(); + + // If we have multiple cockroach-admin servers, and the first N of them + // don't respond successfully, we should keep trying until we find one + // that does (or they all fail). We'll start with the "all fail" case. + let mut mock_bad1 = httptest::Server::run(); + let mut mock_bad2 = httptest::Server::run(); + let add_decommission_failure_expecation = + move |mock_server: &mut httptest::Server| { + mock_server.expect( + Expectation::matching(all_of![ + request::method_path("POST", "/node/decommission"), + request::body(json_decoded(move |n: &NodeId| { + n.node_id == crdb_node_id + })) + ]) + .respond_with(status_code(503)), + ); + }; + add_decommission_failure_expecation(&mut mock_bad1); + add_decommission_failure_expecation(&mut mock_bad2); + let mut fake_resolver = + FixedResolver(vec![mock_bad1.addr(), mock_bad2.addr()]); + let mut err = clean_up_expunged_zones( + &opctx, + datastore, + &fake_resolver, + iter::once((any_sled_id, &crdb_zone)), + ) + .await + .expect_err("no successful response should result in failure"); + assert_eq!(err.len(), 1); + let err = err.pop().unwrap(); + assert_eq!( + err.to_string(), + format!( + "failed to decommission cockroach zone {} \ + (node {crdb_node_id}): failed to contact 2 admin servers", + crdb_zone.id + ) + ); + mock_bad1.verify_and_clear(); + mock_bad2.verify_and_clear(); + mock_admin.verify_and_clear(); + + // Now we try again, but put the good server at the end of the list; we + // should contact the two bad servers, but then succeed on the good one. + add_decommission_failure_expecation(&mut mock_bad1); + add_decommission_failure_expecation(&mut mock_bad2); + add_decommission_expecation(&mut mock_admin); + fake_resolver.0.push(mock_admin.addr()); + clean_up_expunged_zones( + &opctx, + datastore, + &fake_resolver, + iter::once((any_sled_id, &crdb_zone)), + ) + .await + .expect("decommissioned test node"); + mock_bad1.verify_and_clear(); + mock_bad2.verify_and_clear(); + mock_admin.verify_and_clear(); + } } diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index dddc44ce86..b56f518a24 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -423,6 +423,7 @@ impl BackgroundTasksInitializer { // Background task: blueprint executor let blueprint_executor = blueprint_execution::BlueprintExecutor::new( datastore.clone(), + resolver.clone(), rx_blueprint.clone(), nexus_id.to_string(), ); diff --git a/nexus/src/app/background/tasks/blueprint_execution.rs b/nexus/src/app/background/tasks/blueprint_execution.rs index 451317f42a..16bf872f2a 100644 --- a/nexus/src/app/background/tasks/blueprint_execution.rs +++ b/nexus/src/app/background/tasks/blueprint_execution.rs @@ -7,6 +7,7 @@ use crate::app::background::BackgroundTask; use futures::future::BoxFuture; use futures::FutureExt; +use internal_dns::resolver::Resolver; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::{Blueprint, BlueprintTarget}; @@ -18,6 +19,7 @@ use tokio::sync::watch; /// the state of the system based on the `Blueprint`. pub struct BlueprintExecutor { datastore: Arc, + resolver: Resolver, rx_blueprint: watch::Receiver>>, nexus_label: String, tx: watch::Sender, @@ -26,13 +28,14 @@ pub struct BlueprintExecutor { impl BlueprintExecutor { pub fn new( datastore: Arc, + resolver: Resolver, rx_blueprint: watch::Receiver< Option>, >, nexus_label: String, ) -> BlueprintExecutor { let (tx, _) = watch::channel(0); - BlueprintExecutor { datastore, rx_blueprint, nexus_label, tx } + BlueprintExecutor { datastore, resolver, rx_blueprint, nexus_label, tx } } pub fn watcher(&self) -> watch::Receiver { @@ -76,6 +79,7 @@ impl BlueprintExecutor { let result = nexus_reconfigurator_execution::realize_blueprint( opctx, &self.datastore, + &self.resolver, blueprint, &self.nexus_label, ) @@ -186,6 +190,7 @@ mod test { // Set up the test. let nexus = &cptestctx.server.server_context().nexus; let datastore = nexus.datastore(); + let resolver = nexus.resolver(); let opctx = OpContext::for_background( cptestctx.logctx.log.clone(), nexus.authz.clone(), @@ -232,6 +237,7 @@ mod test { let (blueprint_tx, blueprint_rx) = watch::channel(None); let mut task = BlueprintExecutor::new( datastore.clone(), + resolver.clone(), blueprint_rx, String::from("test-suite"), ); diff --git a/nexus/src/app/background/tasks/crdb_node_id_collector.rs b/nexus/src/app/background/tasks/crdb_node_id_collector.rs index 0da411699e..d33dfe2634 100644 --- a/nexus/src/app/background/tasks/crdb_node_id_collector.rs +++ b/nexus/src/app/background/tasks/crdb_node_id_collector.rs @@ -185,7 +185,7 @@ async fn ensure_node_id_known( let admin_client = cockroach_admin_client::Client::new(&admin_url, opctx.log.clone()); let node = admin_client - .node_id() + .local_node_id() .await .with_context(|| { format!("failed to fetch node ID for zone {zone_id} at {admin_url}") @@ -502,7 +502,7 @@ mod tests { // Node 1 succeeds. admin1.expect(Expectation::matching(any()).times(1).respond_with( - json_encoded(cockroach_admin_client::types::NodeId { + json_encoded(cockroach_admin_client::types::LocalNodeId { zone_id: crdb_zone_id1, node_id: crdb_node_id1.to_string(), }), @@ -515,7 +515,7 @@ mod tests { ); // Node 3 succeeds, but with an unexpected zone_id. admin3.expect(Expectation::matching(any()).times(1).respond_with( - json_encoded(cockroach_admin_client::types::NodeId { + json_encoded(cockroach_admin_client::types::LocalNodeId { zone_id: crdb_zone_id4, node_id: crdb_node_id3.to_string(), }), diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 83e72198b9..a41fa0bd4e 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1546,7 +1546,7 @@ impl super::Nexus { ) -> Result<(), Error> { notify_instance_updated( &self.datastore(), - &self.resolver().await, + self.resolver(), &self.opctx_alloc, opctx, &self.log, diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs index f0d6717392..5f5274dea2 100644 --- a/nexus/src/app/instance_network.rs +++ b/nexus/src/app/instance_network.rs @@ -74,7 +74,7 @@ impl Nexus { instance_ensure_dpd_config( &self.db_datastore, &self.log, - &self.resolver().await, + self.resolver(), opctx, &self.opctx_alloc, instance_id, @@ -135,11 +135,10 @@ impl Nexus { opctx: &OpContext, authz_instance: &authz::Instance, ) -> Result<(), Error> { - let resolver = self.resolver().await; instance_delete_dpd_config( &self.db_datastore, &self.log, - &resolver, + self.resolver(), opctx, &self.opctx_alloc, authz_instance, @@ -178,7 +177,7 @@ impl Nexus { ) -> Result<(), Error> { delete_dpd_config_by_entry( &self.db_datastore, - &self.resolver().await, + self.resolver(), &self.log, opctx, &self.opctx_alloc, @@ -199,7 +198,7 @@ impl Nexus { probe_delete_dpd_config( &self.db_datastore, &self.log, - &self.resolver().await, + self.resolver(), opctx, &self.opctx_alloc, probe_id, diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 3202b105c5..4b8d148d7b 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -934,8 +934,8 @@ impl Nexus { *mid } - pub(crate) async fn resolver(&self) -> internal_dns::resolver::Resolver { - self.internal_resolver.clone() + pub fn resolver(&self) -> &internal_dns::resolver::Resolver { + &self.internal_resolver } /// Reliable persistent workflows can request that sagas be executed by @@ -1030,16 +1030,16 @@ impl Nexus { pub(crate) async fn dpd_clients( &self, ) -> Result, String> { - let resolver = self.resolver().await; - dpd_clients(&resolver, &self.log).await + let resolver = self.resolver(); + dpd_clients(resolver, &self.log).await } pub(crate) async fn mg_clients( &self, ) -> Result, String> { - let resolver = self.resolver().await; + let resolver = self.resolver(); let mappings = - switch_zone_address_mappings(&resolver, &self.log).await?; + switch_zone_address_mappings(resolver, &self.log).await?; let mut clients: Vec<(SwitchLocation, mg_admin_client::Client)> = vec![]; for (location, addr) in &mappings { diff --git a/nexus/src/app/sagas/common_storage.rs b/nexus/src/app/sagas/common_storage.rs index 44dd72c571..eaf144eaa9 100644 --- a/nexus/src/app/sagas/common_storage.rs +++ b/nexus/src/app/sagas/common_storage.rs @@ -27,7 +27,6 @@ pub(crate) async fn get_pantry_address( ) -> Result { nexus .resolver() - .await .lookup_socket_v6(ServiceName::CruciblePantry) .await .map_err(|e| e.to_string()) diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index 4f1984db7f..4e655a1ed0 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -934,6 +934,7 @@ impl BlueprintZoneDisposition { match self { Self::InService => match filter { BlueprintZoneFilter::All => true, + BlueprintZoneFilter::Expunged => false, BlueprintZoneFilter::ShouldBeRunning => true, BlueprintZoneFilter::ShouldBeExternallyReachable => true, BlueprintZoneFilter::ShouldBeInInternalDns => true, @@ -941,6 +942,7 @@ impl BlueprintZoneDisposition { }, Self::Quiesced => match filter { BlueprintZoneFilter::All => true, + BlueprintZoneFilter::Expunged => false, // Quiesced zones are still running. BlueprintZoneFilter::ShouldBeRunning => true, @@ -957,6 +959,7 @@ impl BlueprintZoneDisposition { }, Self::Expunged => match filter { BlueprintZoneFilter::All => true, + BlueprintZoneFilter::Expunged => true, BlueprintZoneFilter::ShouldBeRunning => false, BlueprintZoneFilter::ShouldBeExternallyReachable => false, BlueprintZoneFilter::ShouldBeInInternalDns => false, @@ -1001,6 +1004,9 @@ pub enum BlueprintZoneFilter { /// All zones. All, + /// Zones that have been expunged. + Expunged, + /// Zones that are desired to be in the RUNNING state ShouldBeRunning, diff --git a/openapi/cockroach-admin.json b/openapi/cockroach-admin.json index a46b0014a1..3b03475ec5 100644 --- a/openapi/cockroach-admin.json +++ b/openapi/cockroach-admin.json @@ -10,17 +10,51 @@ "version": "0.0.1" }, "paths": { + "/node/decommission": { + "post": { + "summary": "Decommission a node from the CRDB cluster", + "operationId": "node_decommission", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NodeId" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NodeDecommission" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/node/id": { "get": { "summary": "Get the CockroachDB node ID of the local cockroach instance.", - "operationId": "node_id", + "operationId": "local_node_id", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/NodeId" + "$ref": "#/components/schemas/LocalNodeId" } } } @@ -94,7 +128,7 @@ "request_id" ] }, - "NodeId": { + "LocalNodeId": { "description": "CockroachDB Node ID", "type": "object", "properties": { @@ -115,6 +149,120 @@ "zone_id" ] }, + "NodeDecommission": { + "type": "object", + "properties": { + "is_decommissioning": { + "type": "boolean" + }, + "is_draining": { + "type": "boolean" + }, + "is_live": { + "type": "boolean" + }, + "membership": { + "$ref": "#/components/schemas/NodeMembership" + }, + "node_id": { + "type": "string" + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "replicas": { + "type": "integer", + "format": "int64" + } + }, + "required": [ + "is_decommissioning", + "is_draining", + "is_live", + "membership", + "node_id", + "notes", + "replicas" + ] + }, + "NodeId": { + "type": "object", + "properties": { + "node_id": { + "type": "string" + } + }, + "required": [ + "node_id" + ] + }, + "NodeMembership": { + "oneOf": [ + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "active" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "decommissioning" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "decommissioned" + ] + } + }, + "required": [ + "state" + ] + }, + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "unknown" + ] + }, + "value": { + "type": "string" + } + }, + "required": [ + "state", + "value" + ] + } + ] + }, "NodeStatus": { "type": "object", "properties": {