Add omdb sled-agent bootstore status (#5041)

Fixes #4295 and #3722

bootstore/src/schemes/v0/peer.rs

@@ -62,6 +62,14 @@ pub enum NodeRequestError {
     },
 }
 
+impl From<NodeRequestError> for omicron_common::api::external::Error {
+    fn from(error: NodeRequestError) -> Self {
+        omicron_common::api::external::Error::internal_error(&format!(
+            "{error}"
+        ))
+    }
+}
+
 /// A request sent to the `Node` task from the `NodeHandle`
 pub enum NodeApiRequest {
     /// Initialize a rack at the behest of RSS running on the same scrimlet as

dev-tools/omdb/src/bin/omdb/sled_agent.rs

@@ -31,6 +31,10 @@ enum SledAgentCommands {
     /// print information about zpools
     #[clap(subcommand)]
     Zpools(ZpoolCommands),
+
+    /// print information about the local bootstore node
+    #[clap(subcommand)]
+    Bootstore(BootstoreCommands),
 }
 
 #[derive(Debug, Subcommand)]
@@ -45,6 +49,12 @@ enum ZpoolCommands {
     List,
 }
 
+#[derive(Debug, Subcommand)]
+enum BootstoreCommands {
+    /// Show the internal state of the local bootstore node
+    Status,
+}
+
 impl SledAgentArgs {
     /// Run a `omdb sled-agent` subcommand.
     pub(crate) async fn run_cmd(
@@ -70,6 +80,9 @@ impl SledAgentArgs {
             SledAgentCommands::Zpools(ZpoolCommands::List) => {
                 cmd_zpools_list(&client).await
             }
+            SledAgentCommands::Bootstore(BootstoreCommands::Status) => {
+                cmd_bootstore_status(&client).await
+            }
         }
     }
 }
@@ -110,3 +123,46 @@ async fn cmd_zpools_list(
 
     Ok(())
 }
+
+/// Runs `omdb sled-agent bootstore status`
+async fn cmd_bootstore_status(
+    client: &sled_agent_client::Client,
+) -> Result<(), anyhow::Error> {
+    let status = client.bootstore_status().await.context("bootstore status")?;
+    println!("fsm ledger generation: {}", status.fsm_ledger_generation);
+    println!(
+        "network config ledger generation: {:?}",
+        status.network_config_ledger_generation
+    );
+    println!("fsm state: {}", status.fsm_state);
+    println!("peers (found by ddmd):");
+    if status.peers.is_empty() {
+        println!("    <none>");
+    }
+    for peer in status.peers.iter() {
+        println!("    {peer}");
+    }
+    println!("established connections:");
+    if status.established_connections.is_empty() {
+        println!("    <none>");
+    }
+    for c in status.established_connections.iter() {
+        println!("     {:?} : {}", c.baseboard, c.addr);
+    }
+    println!("accepted connections:");
+    if status.accepted_connections.is_empty() {
+        println!("    <none>");
+    }
+    for addr in status.accepted_connections.iter() {
+        println!("    {addr}");
+    }
+    println!("negotiating connections:");
+    if status.negotiating_connections.is_empty() {
+        println!("    <none>");
+    }
+    for addr in status.negotiating_connections.iter() {
+        println!("    {addr}");
+    }
+
+    Ok(())
+}

dev-tools/omdb/tests/usage_errors.out

@@ -331,9 +331,10 @@ Debug a specific Sled
 Usage: omdb sled-agent [OPTIONS] <COMMAND>
 
 Commands:
-  zones   print information about zones
-  zpools  print information about zpools
-  help    Print this message or the help of the given subcommand(s)
+  zones      print information about zones
+  zpools     print information about zpools
+  bootstore  print information about the local bootstore node
+  help       Print this message or the help of the given subcommand(s)
 
 Options:
       --sled-agent-url <SLED_AGENT_URL>  URL of the Sled internal API [env: OMDB_SLED_AGENT_URL=]

openapi/sled-agent.json

@@ -136,6 +136,30 @@
         }
       }
     },
+    "/bootstore/status": {
+      "get": {
+        "summary": "Get the internal state of the local bootstore node",
+        "operationId": "bootstore_status",
+        "responses": {
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/BootstoreStatus"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "$ref": "#/components/responses/Error"
+          },
+          "5XX": {
+            "$ref": "#/components/responses/Error"
+          }
+        }
+      }
+    },
     "/cockroachdb": {
       "post": {
         "summary": "Initializes a CockroachDB cluster",
@@ -2531,6 +2555,60 @@
           }
         ]
       },
+      "BootstoreStatus": {
+        "type": "object",
+        "properties": {
+          "accepted_connections": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "uniqueItems": true
+          },
+          "established_connections": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/EstablishedConnection"
+            }
+          },
+          "fsm_ledger_generation": {
+            "type": "integer",
+            "format": "uint64",
+            "minimum": 0
+          },
+          "fsm_state": {
+            "type": "string"
+          },
+          "negotiating_connections": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "uniqueItems": true
+          },
+          "network_config_ledger_generation": {
+            "nullable": true,
+            "type": "integer",
+            "format": "uint64",
+            "minimum": 0
+          },
+          "peers": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "uniqueItems": true
+          }
+        },
+        "required": [
+          "accepted_connections",
+          "established_connections",
+          "fsm_ledger_generation",
+          "fsm_state",
+          "negotiating_connections",
+          "peers"
+        ]
+      },
       "BundleUtilization": {
         "description": "The portion of a debug dataset used for zone bundles.",
         "type": "object",
@@ -3847,6 +3925,21 @@
           "request_id"
         ]
       },
+      "EstablishedConnection": {
+        "type": "object",
+        "properties": {
+          "addr": {
+            "type": "string"
+          },
+          "baseboard": {
+            "$ref": "#/components/schemas/Baseboard"
+          }
+        },
+        "required": [
+          "addr",
+          "baseboard"
+        ]
+      },
       "Field": {
         "description": "A `Field` is a named aspect of a target or metric.",
         "type": "object",

sled-agent/src/http_entrypoints.rs

@@ -8,7 +8,7 @@ use super::sled_agent::SledAgent;
 use crate::bootstrap::early_networking::EarlyNetworkConfig;
 use crate::bootstrap::params::AddSledRequest;
 use crate::params::{
-    CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody,
+    BootstoreStatus, CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody,
     InstanceExternalIpBody, InstancePutMigrationIdsBody, InstancePutStateBody,
     InstancePutStateResponse, InstanceUnregisterResponse, Inventory,
     OmicronZonesConfig, SledRole, TimeSync, VpcFirewallRulesEnsureBody,
@@ -85,6 +85,7 @@ pub fn api() -> SledApiDescription {
         api.register(host_os_write_status_get)?;
         api.register(host_os_write_status_delete)?;
         api.register(inventory)?;
+        api.register(bootstore_status)?;
 
         Ok(())
     }
@@ -972,3 +973,23 @@ async fn inventory(
     let sa = request_context.context();
     Ok(HttpResponseOk(sa.inventory()?))
 }
+
+/// Get the internal state of the local bootstore node
+#[endpoint {
+    method = GET,
+    path = "/bootstore/status",
+}]
+async fn bootstore_status(
+    request_context: RequestContext<SledAgent>,
+) -> Result<HttpResponseOk<BootstoreStatus>, HttpError> {
+    let sa = request_context.context();
+    let bootstore = sa.bootstore();
+    let status = bootstore
+        .get_status()
+        .await
+        .map_err(|e| {
+            HttpError::from(omicron_common::api::external::Error::from(e))
+        })?
+        .into();
+    Ok(HttpResponseOk(status))
+}

sled-agent/src/params.rs

@@ -25,6 +25,7 @@ use sled_hardware::Baseboard;
 pub use sled_hardware::DendriteAsic;
 use sled_storage::dataset::DatasetKind;
 use sled_storage::dataset::DatasetName;
+use std::collections::BTreeSet;
 use std::fmt::{Debug, Display, Formatter, Result as FormatResult};
 use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6};
 use std::str::FromStr;
@@ -865,3 +866,45 @@ pub struct Inventory {
     pub usable_physical_ram: ByteCount,
     pub reservoir_size: ByteCount,
 }
+
+#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
+pub struct EstablishedConnection {
+    baseboard: Baseboard,
+    addr: SocketAddrV6,
+}
+
+impl From<(Baseboard, SocketAddrV6)> for EstablishedConnection {
+    fn from(value: (Baseboard, SocketAddrV6)) -> Self {
+        EstablishedConnection { baseboard: value.0, addr: value.1 }
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)]
+pub struct BootstoreStatus {
+    pub fsm_ledger_generation: u64,
+    pub network_config_ledger_generation: Option<u64>,
+    pub fsm_state: String,
+    pub peers: BTreeSet<SocketAddrV6>,
+    pub established_connections: Vec<EstablishedConnection>,
+    pub accepted_connections: BTreeSet<SocketAddrV6>,
+    pub negotiating_connections: BTreeSet<SocketAddrV6>,
+}
+
+impl From<bootstore::schemes::v0::Status> for BootstoreStatus {
+    fn from(value: bootstore::schemes::v0::Status) -> Self {
+        BootstoreStatus {
+            fsm_ledger_generation: value.fsm_ledger_generation,
+            network_config_ledger_generation: value
+                .network_config_ledger_generation,
+            fsm_state: value.fsm_state.to_string(),
+            peers: value.peers,
+            established_connections: value
+                .connections
+                .into_iter()
+                .map(EstablishedConnection::from)
+                .collect(),
+            accepted_connections: value.accepted_connections,
+            negotiating_connections: value.negotiating_connections,
+        }
+    }
+}