From 6c9c9b73d82c91389f0c8ccb46e33aaaf0afdf03 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Tue, 13 Feb 2024 14:58:05 -0500 Subject: [PATCH] Add `omdb sled-agent bootstore status` (#5041) Fixes #4295 and #3722 --- bootstore/src/schemes/v0/peer.rs | 8 ++ dev-tools/omdb/src/bin/omdb/sled_agent.rs | 56 ++++++++++++++ dev-tools/omdb/tests/usage_errors.out | 7 +- openapi/sled-agent.json | 93 +++++++++++++++++++++++ sled-agent/src/http_entrypoints.rs | 23 +++++- sled-agent/src/params.rs | 43 +++++++++++ 6 files changed, 226 insertions(+), 4 deletions(-) diff --git a/bootstore/src/schemes/v0/peer.rs b/bootstore/src/schemes/v0/peer.rs index 5290e64672..1175e64143 100644 --- a/bootstore/src/schemes/v0/peer.rs +++ b/bootstore/src/schemes/v0/peer.rs @@ -62,6 +62,14 @@ pub enum NodeRequestError { }, } +impl From for omicron_common::api::external::Error { + fn from(error: NodeRequestError) -> Self { + omicron_common::api::external::Error::internal_error(&format!( + "{error}" + )) + } +} + /// A request sent to the `Node` task from the `NodeHandle` pub enum NodeApiRequest { /// Initialize a rack at the behest of RSS running on the same scrimlet as diff --git a/dev-tools/omdb/src/bin/omdb/sled_agent.rs b/dev-tools/omdb/src/bin/omdb/sled_agent.rs index c413a2ba43..2d9e19d253 100644 --- a/dev-tools/omdb/src/bin/omdb/sled_agent.rs +++ b/dev-tools/omdb/src/bin/omdb/sled_agent.rs @@ -31,6 +31,10 @@ enum SledAgentCommands { /// print information about zpools #[clap(subcommand)] Zpools(ZpoolCommands), + + /// print information about the local bootstore node + #[clap(subcommand)] + Bootstore(BootstoreCommands), } #[derive(Debug, Subcommand)] @@ -45,6 +49,12 @@ enum ZpoolCommands { List, } +#[derive(Debug, Subcommand)] +enum BootstoreCommands { + /// Show the internal state of the local bootstore node + Status, +} + impl SledAgentArgs { /// Run a `omdb sled-agent` subcommand. pub(crate) async fn run_cmd( @@ -70,6 +80,9 @@ impl SledAgentArgs { SledAgentCommands::Zpools(ZpoolCommands::List) => { cmd_zpools_list(&client).await } + SledAgentCommands::Bootstore(BootstoreCommands::Status) => { + cmd_bootstore_status(&client).await + } } } } @@ -110,3 +123,46 @@ async fn cmd_zpools_list( Ok(()) } + +/// Runs `omdb sled-agent bootstore status` +async fn cmd_bootstore_status( + client: &sled_agent_client::Client, +) -> Result<(), anyhow::Error> { + let status = client.bootstore_status().await.context("bootstore status")?; + println!("fsm ledger generation: {}", status.fsm_ledger_generation); + println!( + "network config ledger generation: {:?}", + status.network_config_ledger_generation + ); + println!("fsm state: {}", status.fsm_state); + println!("peers (found by ddmd):"); + if status.peers.is_empty() { + println!(" "); + } + for peer in status.peers.iter() { + println!(" {peer}"); + } + println!("established connections:"); + if status.established_connections.is_empty() { + println!(" "); + } + for c in status.established_connections.iter() { + println!(" {:?} : {}", c.baseboard, c.addr); + } + println!("accepted connections:"); + if status.accepted_connections.is_empty() { + println!(" "); + } + for addr in status.accepted_connections.iter() { + println!(" {addr}"); + } + println!("negotiating connections:"); + if status.negotiating_connections.is_empty() { + println!(" "); + } + for addr in status.negotiating_connections.iter() { + println!(" {addr}"); + } + + Ok(()) +} diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index 6f9b539371..bb7da1be57 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -331,9 +331,10 @@ Debug a specific Sled Usage: omdb sled-agent [OPTIONS] Commands: - zones print information about zones - zpools print information about zpools - help Print this message or the help of the given subcommand(s) + zones print information about zones + zpools print information about zpools + bootstore print information about the local bootstore node + help Print this message or the help of the given subcommand(s) Options: --sled-agent-url URL of the Sled internal API [env: OMDB_SLED_AGENT_URL=] diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 395394defb..99156fffd4 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -136,6 +136,30 @@ } } }, + "/bootstore/status": { + "get": { + "summary": "Get the internal state of the local bootstore node", + "operationId": "bootstore_status", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BootstoreStatus" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/cockroachdb": { "post": { "summary": "Initializes a CockroachDB cluster", @@ -2531,6 +2555,60 @@ } ] }, + "BootstoreStatus": { + "type": "object", + "properties": { + "accepted_connections": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "established_connections": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EstablishedConnection" + } + }, + "fsm_ledger_generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "fsm_state": { + "type": "string" + }, + "negotiating_connections": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "network_config_ledger_generation": { + "nullable": true, + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "peers": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": [ + "accepted_connections", + "established_connections", + "fsm_ledger_generation", + "fsm_state", + "negotiating_connections", + "peers" + ] + }, "BundleUtilization": { "description": "The portion of a debug dataset used for zone bundles.", "type": "object", @@ -3847,6 +3925,21 @@ "request_id" ] }, + "EstablishedConnection": { + "type": "object", + "properties": { + "addr": { + "type": "string" + }, + "baseboard": { + "$ref": "#/components/schemas/Baseboard" + } + }, + "required": [ + "addr", + "baseboard" + ] + }, "Field": { "description": "A `Field` is a named aspect of a target or metric.", "type": "object", diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 0798aed664..5f888504db 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -8,7 +8,7 @@ use super::sled_agent::SledAgent; use crate::bootstrap::early_networking::EarlyNetworkConfig; use crate::bootstrap::params::AddSledRequest; use crate::params::{ - CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody, + BootstoreStatus, CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody, InstanceExternalIpBody, InstancePutMigrationIdsBody, InstancePutStateBody, InstancePutStateResponse, InstanceUnregisterResponse, Inventory, OmicronZonesConfig, SledRole, TimeSync, VpcFirewallRulesEnsureBody, @@ -85,6 +85,7 @@ pub fn api() -> SledApiDescription { api.register(host_os_write_status_get)?; api.register(host_os_write_status_delete)?; api.register(inventory)?; + api.register(bootstore_status)?; Ok(()) } @@ -972,3 +973,23 @@ async fn inventory( let sa = request_context.context(); Ok(HttpResponseOk(sa.inventory()?)) } + +/// Get the internal state of the local bootstore node +#[endpoint { + method = GET, + path = "/bootstore/status", +}] +async fn bootstore_status( + request_context: RequestContext, +) -> Result, HttpError> { + let sa = request_context.context(); + let bootstore = sa.bootstore(); + let status = bootstore + .get_status() + .await + .map_err(|e| { + HttpError::from(omicron_common::api::external::Error::from(e)) + })? + .into(); + Ok(HttpResponseOk(status)) +} diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 52bfb20e5d..7ed1264d9c 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -25,6 +25,7 @@ use sled_hardware::Baseboard; pub use sled_hardware::DendriteAsic; use sled_storage::dataset::DatasetKind; use sled_storage::dataset::DatasetName; +use std::collections::BTreeSet; use std::fmt::{Debug, Display, Formatter, Result as FormatResult}; use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; use std::str::FromStr; @@ -865,3 +866,45 @@ pub struct Inventory { pub usable_physical_ram: ByteCount, pub reservoir_size: ByteCount, } + +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct EstablishedConnection { + baseboard: Baseboard, + addr: SocketAddrV6, +} + +impl From<(Baseboard, SocketAddrV6)> for EstablishedConnection { + fn from(value: (Baseboard, SocketAddrV6)) -> Self { + EstablishedConnection { baseboard: value.0, addr: value.1 } + } +} + +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct BootstoreStatus { + pub fsm_ledger_generation: u64, + pub network_config_ledger_generation: Option, + pub fsm_state: String, + pub peers: BTreeSet, + pub established_connections: Vec, + pub accepted_connections: BTreeSet, + pub negotiating_connections: BTreeSet, +} + +impl From for BootstoreStatus { + fn from(value: bootstore::schemes::v0::Status) -> Self { + BootstoreStatus { + fsm_ledger_generation: value.fsm_ledger_generation, + network_config_ledger_generation: value + .network_config_ledger_generation, + fsm_state: value.fsm_state.to_string(), + peers: value.peers, + established_connections: value + .connections + .into_iter() + .map(EstablishedConnection::from) + .collect(), + accepted_connections: value.accepted_connections, + negotiating_connections: value.negotiating_connections, + } + } +}