diff --git a/Cargo.lock b/Cargo.lock index 15496909d2..3cc7d09d82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4230,12 +4230,18 @@ dependencies = [ "base64", "chrono", "expectorate", + "futures", "gateway-client", "gateway-messages", "gateway-test-utils", "nexus-types", + "omicron-common", + "omicron-sled-agent", "omicron-workspace-hack", "regex", + "reqwest", + "serde_json", + "sled-agent-client", "slog", "strum", "thiserror", @@ -4321,6 +4327,7 @@ dependencies = [ "schemars", "serde", "serde_json", + "sled-agent-client", "steno", "strum", "uuid", diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 0bbd27cf3e..89f41d10a6 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -10,7 +10,7 @@ use uuid::Uuid; progenitor::generate_api!( spec = "../../openapi/sled-agent.json", - derives = [ schemars::JsonSchema ], + derives = [ schemars::JsonSchema, PartialEq ], inner_type = slog::Logger, pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { slog::debug!(log, "client request"; @@ -25,7 +25,6 @@ progenitor::generate_api!( //TODO trade the manual transformations later in this file for the // replace directives below? replace = { - //Ipv4Network = ipnetwork::Ipv4Network, SwitchLocation = omicron_common::api::external::SwitchLocation, Ipv6Network = ipnetwork::Ipv6Network, IpNetwork = ipnetwork::IpNetwork, @@ -34,6 +33,37 @@ progenitor::generate_api!( } ); +// We cannot easily configure progenitor to derive `Eq` on all the client- +// generated types because some have floats and other types that can't impl +// `Eq`. We impl it explicitly for a few types on which we need it. +impl Eq for types::OmicronZonesConfig {} +impl Eq for types::OmicronZoneConfig {} +impl Eq for types::OmicronZoneType {} +impl Eq for types::OmicronZoneDataset {} + +impl types::OmicronZoneType { + /// Human-readable label describing what kind of zone this is + /// + /// This is just use for testing and reporting. + pub fn label(&self) -> impl std::fmt::Display { + match self { + types::OmicronZoneType::BoundaryNtp { .. } => "boundary_ntp", + types::OmicronZoneType::Clickhouse { .. } => "clickhouse", + types::OmicronZoneType::ClickhouseKeeper { .. } => { + "clickhouse_keeper" + } + types::OmicronZoneType::CockroachDb { .. } => "cockroach_db", + types::OmicronZoneType::Crucible { .. } => "crucible", + types::OmicronZoneType::CruciblePantry { .. } => "crucible_pantry", + types::OmicronZoneType::ExternalDns { .. } => "external_dns", + types::OmicronZoneType::InternalDns { .. } => "internal_dns", + types::OmicronZoneType::InternalNtp { .. } => "internal_ntp", + types::OmicronZoneType::Nexus { .. } => "nexus", + types::OmicronZoneType::Oximeter { .. } => "oximeter", + } + } +} + impl omicron_common::api::external::ClientError for types::Error { fn message(&self) -> String { self.message.clone() @@ -245,6 +275,12 @@ impl From<&omicron_common::api::external::Name> for types::Name { } } +impl From for omicron_common::api::external::Name { + fn from(s: types::Name) -> Self { + Self::try_from(s.as_str().to_owned()).unwrap() + } +} + impl From for types::Vni { fn from(v: omicron_common::api::external::Vni) -> Self { Self(u32::from(v)) @@ -264,6 +300,12 @@ impl From for types::MacAddr { } } +impl From for omicron_common::api::external::MacAddr { + fn from(s: types::MacAddr) -> Self { + s.parse().unwrap() + } +} + impl From for types::Ipv4Net { fn from(n: omicron_common::api::external::Ipv4Net) -> Self { Self::try_from(n.to_string()).unwrap_or_else(|e| panic!("{}: {}", n, e)) @@ -292,6 +334,12 @@ impl From for types::Ipv4Net { } } +impl From for ipnetwork::Ipv4Network { + fn from(n: types::Ipv4Net) -> Self { + n.parse().unwrap() + } +} + impl From for types::Ipv4Network { fn from(n: ipnetwork::Ipv4Network) -> Self { Self::try_from(n.to_string()).unwrap_or_else(|e| panic!("{}: {}", n, e)) @@ -304,6 +352,12 @@ impl From for types::Ipv6Net { } } +impl From for ipnetwork::Ipv6Network { + fn from(n: types::Ipv6Net) -> Self { + n.parse().unwrap() + } +} + impl From for types::IpNet { fn from(n: ipnetwork::IpNetwork) -> Self { use ipnetwork::IpNetwork; @@ -314,6 +368,15 @@ impl From for types::IpNet { } } +impl From for ipnetwork::IpNetwork { + fn from(n: types::IpNet) -> Self { + match n { + types::IpNet::V4(v4) => ipnetwork::IpNetwork::V4(v4.into()), + types::IpNet::V6(v6) => ipnetwork::IpNetwork::V6(v6.into()), + } + } +} + impl From for types::Ipv4Net { fn from(n: std::net::Ipv4Addr) -> Self { Self::try_from(format!("{n}/32")) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 446152137a..3b05c58df3 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -525,7 +525,9 @@ impl JsonSchema for RoleName { // to serialize the value. // // TODO: custom JsonSchema and Deserialize impls to enforce i64::MAX limit -#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +#[derive( + Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, +)] pub struct ByteCount(u64); #[allow(non_upper_case_globals)] diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index 08a783d8c8..ad7ab35455 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -2472,6 +2472,7 @@ async fn cmd_db_inventory_collections_show( inv_collection_print(&collection).await?; let nerrors = inv_collection_print_errors(&collection).await?; inv_collection_print_devices(&collection, &long_string_formatter).await?; + inv_collection_print_sleds(&collection); if nerrors > 0 { eprintln!( @@ -2703,6 +2704,58 @@ async fn inv_collection_print_devices( Ok(()) } +fn inv_collection_print_sleds(collection: &Collection) { + println!("SLED AGENTS"); + for sled in collection.sled_agents.values() { + println!( + "\nsled {} (role = {:?}, serial {})", + sled.sled_id, + sled.sled_role, + match &sled.baseboard_id { + Some(baseboard_id) => &baseboard_id.serial_number, + None => "unknown", + }, + ); + println!( + " found at: {} from {}", + sled.time_collected, sled.source + ); + println!(" address: {}", sled.sled_agent_address); + println!(" usable hw threads: {}", sled.usable_hardware_threads); + println!( + " usable memory (GiB): {}", + sled.usable_physical_ram.to_whole_gibibytes() + ); + println!( + " reservoir (GiB): {}", + sled.reservoir_size.to_whole_gibibytes() + ); + + if let Some(zones) = collection.omicron_zones.get(&sled.sled_id) { + println!( + " zones collected from {} at {}", + zones.source, zones.time_collected, + ); + println!( + " zones generation: {} (count: {})", + *zones.zones.generation, + zones.zones.zones.len() + ); + + if zones.zones.zones.is_empty() { + continue; + } + + println!(" ZONES FOUND"); + for z in &zones.zones.zones { + println!(" zone {} (type {})", z.id, z.zone_type.label()); + } + } else { + println!(" warning: no zone information found"); + } + } +} + #[derive(Debug)] struct LongStringFormatter { show_long_strings: bool, diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs index 72671fde98..47e2033718 100644 --- a/nexus/db-model/src/inventory.rs +++ b/nexus/db-model/src/inventory.rs @@ -6,10 +6,16 @@ use crate::schema::{ hw_baseboard_id, inv_caboose, inv_collection, inv_collection_error, - inv_root_of_trust, inv_root_of_trust_page, inv_service_processor, - sw_caboose, sw_root_of_trust_page, + inv_omicron_zone, inv_omicron_zone_nic, inv_root_of_trust, + inv_root_of_trust_page, inv_service_processor, inv_sled_agent, + inv_sled_omicron_zones, sw_caboose, sw_root_of_trust_page, }; -use crate::{impl_enum_type, SqlU16, SqlU32}; +use crate::{ + impl_enum_type, ipv6, ByteCount, Generation, MacAddr, Name, SqlU16, SqlU32, + SqlU8, +}; +use anyhow::{anyhow, ensure}; +use anyhow::{bail, Context}; use chrono::DateTime; use chrono::Utc; use diesel::backend::Backend; @@ -18,9 +24,12 @@ use diesel::expression::AsExpression; use diesel::pg::Pg; use diesel::serialize::ToSql; use diesel::{serialize, sql_types}; +use ipnetwork::IpNetwork; use nexus_types::inventory::{ - BaseboardId, Caboose, Collection, PowerState, RotPage, RotSlot, + BaseboardId, Caboose, Collection, OmicronZoneType, PowerState, RotPage, + RotSlot, }; +use std::net::SocketAddrV6; use uuid::Uuid; // See [`nexus_types::inventory::PowerState`]. @@ -538,3 +547,619 @@ pub struct InvRotPage { pub which: RotPageWhich, pub sw_root_of_trust_page_id: Uuid, } + +// See [`nexus_types::inventory::SledRole`]. +impl_enum_type!( + #[derive(SqlType, Debug, QueryId)] + #[diesel(postgres_type(name = "sled_role"))] + pub struct SledRoleEnum; + + #[derive( + Copy, + Clone, + Debug, + AsExpression, + FromSqlRow, + PartialOrd, + Ord, + PartialEq, + Eq + )] + #[diesel(sql_type = SledRoleEnum)] + pub enum SledRole; + + // Enum values + Gimlet => b"gimlet" + Scrimlet => b"scrimlet" +); + +impl From for SledRole { + fn from(value: nexus_types::inventory::SledRole) -> Self { + match value { + nexus_types::inventory::SledRole::Gimlet => SledRole::Gimlet, + nexus_types::inventory::SledRole::Scrimlet => SledRole::Scrimlet, + } + } +} + +impl From for nexus_types::inventory::SledRole { + fn from(value: SledRole) -> Self { + match value { + SledRole::Gimlet => nexus_types::inventory::SledRole::Gimlet, + SledRole::Scrimlet => nexus_types::inventory::SledRole::Scrimlet, + } + } +} + +/// See [`nexus_types::inventory::SledAgent`]. +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = inv_sled_agent)] +pub struct InvSledAgent { + pub inv_collection_id: Uuid, + pub time_collected: DateTime, + pub source: String, + pub sled_id: Uuid, + pub hw_baseboard_id: Option, + pub sled_agent_ip: ipv6::Ipv6Addr, + pub sled_agent_port: SqlU16, + pub sled_role: SledRole, + pub usable_hardware_threads: SqlU32, + pub usable_physical_ram: ByteCount, + pub reservoir_size: ByteCount, +} + +impl InvSledAgent { + pub fn new_without_baseboard( + collection_id: Uuid, + sled_agent: &nexus_types::inventory::SledAgent, + ) -> Result { + // It's irritating to have to check this case at runtime. The challenge + // is that if this sled agent does have a baseboard id, we don't know + // what its (SQL) id is. The only way to get it is to query it from + // the database. As a result, the caller takes a wholly different code + // path for that case that doesn't even involve constructing one of + // these objects. (In fact, we never see the id in Rust.) + // + // To check this at compile time, we'd have to bifurcate + // `nexus_types::inventory::SledAgent` into an enum with two variants: + // one with a baseboard id and one without. This would muck up all the + // other consumers of this type, just for a highly database-specific + // concern. + if sled_agent.baseboard_id.is_some() { + Err(anyhow!( + "attempted to directly insert InvSledAgent with \ + non-null baseboard id" + )) + } else { + Ok(InvSledAgent { + inv_collection_id: collection_id, + time_collected: sled_agent.time_collected, + source: sled_agent.source.clone(), + sled_id: sled_agent.sled_id, + hw_baseboard_id: None, + sled_agent_ip: ipv6::Ipv6Addr::from( + *sled_agent.sled_agent_address.ip(), + ), + sled_agent_port: SqlU16(sled_agent.sled_agent_address.port()), + sled_role: SledRole::from(sled_agent.sled_role), + usable_hardware_threads: SqlU32( + sled_agent.usable_hardware_threads, + ), + usable_physical_ram: ByteCount::from( + sled_agent.usable_physical_ram, + ), + reservoir_size: ByteCount::from(sled_agent.reservoir_size), + }) + } + } +} + +/// See [`nexus_types::inventory::OmicronZonesFound`]. +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = inv_sled_omicron_zones)] +pub struct InvSledOmicronZones { + pub inv_collection_id: Uuid, + pub time_collected: DateTime, + pub source: String, + pub sled_id: Uuid, + pub generation: Generation, +} + +impl InvSledOmicronZones { + pub fn new( + inv_collection_id: Uuid, + zones_found: &nexus_types::inventory::OmicronZonesFound, + ) -> InvSledOmicronZones { + InvSledOmicronZones { + inv_collection_id, + time_collected: zones_found.time_collected, + source: zones_found.source.clone(), + sled_id: zones_found.sled_id, + generation: Generation(zones_found.zones.generation.clone().into()), + } + } + + pub fn into_uninit_zones_found( + self, + ) -> nexus_types::inventory::OmicronZonesFound { + nexus_types::inventory::OmicronZonesFound { + time_collected: self.time_collected, + source: self.source, + sled_id: self.sled_id, + zones: nexus_types::inventory::OmicronZonesConfig { + generation: self.generation.0.into(), + zones: Vec::new(), + }, + } + } +} + +impl_enum_type!( + #[derive(Clone, SqlType, Debug, QueryId)] + #[diesel(postgres_type(name = "zone_type"))] + pub struct ZoneTypeEnum; + + #[derive(Clone, Copy, Debug, Eq, AsExpression, FromSqlRow, PartialEq)] + #[diesel(sql_type = ZoneTypeEnum)] + pub enum ZoneType; + + // Enum values + BoundaryNtp => b"boundary_ntp" + Clickhouse => b"clickhouse" + ClickhouseKeeper => b"clickhouse_keeper" + CockroachDb => b"cockroach_db" + Crucible => b"crucible" + CruciblePantry => b"crucible_pantry" + ExternalDns => b"external_dns" + InternalDns => b"internal_dns" + InternalNtp => b"internal_ntp" + Nexus => b"nexus" + Oximeter => b"oximeter" +); + +/// See [`nexus_types::inventory::OmicronZoneConfig`]. +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = inv_omicron_zone)] +pub struct InvOmicronZone { + pub inv_collection_id: Uuid, + pub sled_id: Uuid, + pub id: Uuid, + pub underlay_address: ipv6::Ipv6Addr, + pub zone_type: ZoneType, + pub primary_service_ip: ipv6::Ipv6Addr, + pub primary_service_port: SqlU16, + pub second_service_ip: Option, + pub second_service_port: Option, + pub dataset_zpool_name: Option, + pub nic_id: Option, + pub dns_gz_address: Option, + pub dns_gz_address_index: Option, + pub ntp_ntp_servers: Option>, + pub ntp_dns_servers: Option>, + pub ntp_domain: Option, + pub nexus_external_tls: Option, + pub nexus_external_dns_servers: Option>, + pub snat_ip: Option, + pub snat_first_port: Option, + pub snat_last_port: Option, +} + +impl InvOmicronZone { + pub fn new( + inv_collection_id: Uuid, + sled_id: Uuid, + zone: &nexus_types::inventory::OmicronZoneConfig, + ) -> Result { + let id = zone.id; + let underlay_address = ipv6::Ipv6Addr::from(zone.underlay_address); + let mut nic_id = None; + let mut dns_gz_address = None; + let mut dns_gz_address_index = None; + let mut ntp_ntp_servers = None; + let mut ntp_dns_servers = None; + let mut ntp_ntp_domain = None; + let mut nexus_external_tls = None; + let mut nexus_external_dns_servers = None; + let mut snat_ip = None; + let mut snat_first_port = None; + let mut snat_last_port = None; + let mut second_service_ip = None; + let mut second_service_port = None; + + let (zone_type, primary_service_sockaddr_str, dataset) = match &zone + .zone_type + { + OmicronZoneType::BoundaryNtp { + address, + ntp_servers, + dns_servers, + domain, + nic, + snat_cfg, + } => { + ntp_ntp_servers = Some(ntp_servers.clone()); + ntp_dns_servers = Some(dns_servers.clone()); + ntp_ntp_domain = domain.clone(); + snat_ip = Some(IpNetwork::from(snat_cfg.ip)); + snat_first_port = Some(SqlU16::from(snat_cfg.first_port)); + snat_last_port = Some(SqlU16::from(snat_cfg.last_port)); + nic_id = Some(nic.id); + (ZoneType::BoundaryNtp, address, None) + } + OmicronZoneType::Clickhouse { address, dataset } => { + (ZoneType::Clickhouse, address, Some(dataset)) + } + OmicronZoneType::ClickhouseKeeper { address, dataset } => { + (ZoneType::ClickhouseKeeper, address, Some(dataset)) + } + OmicronZoneType::CockroachDb { address, dataset } => { + (ZoneType::CockroachDb, address, Some(dataset)) + } + OmicronZoneType::Crucible { address, dataset } => { + (ZoneType::Crucible, address, Some(dataset)) + } + OmicronZoneType::CruciblePantry { address } => { + (ZoneType::CruciblePantry, address, None) + } + OmicronZoneType::ExternalDns { + dataset, + http_address, + dns_address, + nic, + } => { + nic_id = Some(nic.id); + let sockaddr = dns_address + .parse::() + .with_context(|| { + format!( + "parsing address for external DNS server {:?}", + dns_address + ) + })?; + second_service_ip = Some(sockaddr.ip()); + second_service_port = Some(SqlU16::from(sockaddr.port())); + (ZoneType::ExternalDns, http_address, Some(dataset)) + } + OmicronZoneType::InternalDns { + dataset, + http_address, + dns_address, + gz_address, + gz_address_index, + } => { + dns_gz_address = Some(ipv6::Ipv6Addr::from(gz_address)); + dns_gz_address_index = Some(SqlU32::from(*gz_address_index)); + let sockaddr = dns_address + .parse::() + .with_context(|| { + format!( + "parsing address for internal DNS server {:?}", + dns_address + ) + })?; + second_service_ip = Some(sockaddr.ip()); + second_service_port = Some(SqlU16::from(sockaddr.port())); + (ZoneType::InternalDns, http_address, Some(dataset)) + } + OmicronZoneType::InternalNtp { + address, + ntp_servers, + dns_servers, + domain, + } => { + ntp_ntp_servers = Some(ntp_servers.clone()); + ntp_dns_servers = Some(dns_servers.clone()); + ntp_ntp_domain = domain.clone(); + (ZoneType::InternalNtp, address, None) + } + OmicronZoneType::Nexus { + internal_address, + external_ip, + nic, + external_tls, + external_dns_servers, + } => { + nic_id = Some(nic.id); + nexus_external_tls = Some(*external_tls); + nexus_external_dns_servers = Some(external_dns_servers.clone()); + second_service_ip = Some(*external_ip); + (ZoneType::Nexus, internal_address, None) + } + OmicronZoneType::Oximeter { address } => { + (ZoneType::Oximeter, address, None) + } + }; + + let dataset_zpool_name = + dataset.map(|d| d.pool_name.as_str().to_string()); + let primary_service_sockaddr = primary_service_sockaddr_str + .parse::() + .with_context(|| { + format!( + "parsing socket address for primary IP {:?}", + primary_service_sockaddr_str + ) + })?; + let (primary_service_ip, primary_service_port) = ( + ipv6::Ipv6Addr::from(*primary_service_sockaddr.ip()), + SqlU16::from(primary_service_sockaddr.port()), + ); + + Ok(InvOmicronZone { + inv_collection_id, + sled_id, + id, + underlay_address, + zone_type, + primary_service_ip, + primary_service_port, + second_service_ip: second_service_ip.map(IpNetwork::from), + second_service_port, + dataset_zpool_name, + nic_id, + dns_gz_address, + dns_gz_address_index, + ntp_ntp_servers, + ntp_dns_servers: ntp_dns_servers + .map(|list| list.into_iter().map(IpNetwork::from).collect()), + ntp_domain: ntp_ntp_domain, + nexus_external_tls, + nexus_external_dns_servers: nexus_external_dns_servers + .map(|list| list.into_iter().map(IpNetwork::from).collect()), + snat_ip, + snat_first_port, + snat_last_port, + }) + } + + pub fn into_omicron_zone_config( + self, + nic_row: Option, + ) -> Result { + let address = SocketAddrV6::new( + std::net::Ipv6Addr::from(self.primary_service_ip), + *self.primary_service_port, + 0, + 0, + ) + .to_string(); + + // Assemble a value that we can use to extract the NIC _if necessary_ + // and report an error if it was needed but not found. + // + // Any error here should be impossible. By the time we get here, the + // caller should have provided `nic_row` iff there's a corresponding + // `nic_id` in this row, and the ids should match up. And whoever + // created this row ought to have provided a nic_id iff this type of + // zone needs a NIC. This last issue is not under our control, though, + // so we definitely want to handle that as an operational error. The + // others could arguably be programmer errors (i.e., we could `assert`), + // but it seems excessive to crash here. + // + // Note that we immediately return for any of the caller errors here. + // For the other error, we will return only later, if some code path + // below tries to use `nic` when it's not present. + let nic = match (self.nic_id, nic_row) { + (Some(expected_id), Some(nic_row)) => { + ensure!(expected_id == nic_row.id, "caller provided wrong NIC"); + Ok(nic_row.into_network_interface_for_zone(self.id)) + } + (None, None) => Err(anyhow!( + "expected zone to have an associated NIC, but it doesn't" + )), + (Some(_), None) => bail!("caller provided no NIC"), + (None, Some(_)) => bail!("caller unexpectedly provided a NIC"), + }; + + // Similarly, assemble a value that we can use to extract the dataset, + // if necessary. We only return this error if code below tries to use + // this value. + let dataset = self + .dataset_zpool_name + .map(|zpool_name| -> Result<_, anyhow::Error> { + Ok(nexus_types::inventory::OmicronZoneDataset { + pool_name: zpool_name.parse().map_err(|e| { + anyhow!("parsing zpool name {:?}: {}", zpool_name, e) + })?, + }) + }) + .transpose()? + .ok_or_else(|| anyhow!("expected dataset zpool name, found none")); + + // Do the same for the DNS server address. + let dns_address = + match (self.second_service_ip, self.second_service_port) { + (Some(dns_ip), Some(dns_port)) => { + Ok(std::net::SocketAddr::new(dns_ip.ip(), *dns_port) + .to_string()) + } + _ => Err(anyhow!( + "expected second service IP and port, \ + found one missing" + )), + }; + + // Do the same for NTP zone properties. + let ntp_dns_servers = self + .ntp_dns_servers + .ok_or_else(|| anyhow!("expected list of DNS servers, found null")) + .map(|list| { + list.into_iter().map(|ipnetwork| ipnetwork.ip()).collect() + }); + let ntp_ntp_servers = + self.ntp_ntp_servers.ok_or_else(|| anyhow!("expected ntp_servers")); + + let zone_type = match self.zone_type { + ZoneType::BoundaryNtp => { + let snat_cfg = match ( + self.snat_ip, + self.snat_first_port, + self.snat_last_port, + ) { + (Some(ip), Some(first_port), Some(last_port)) => { + nexus_types::inventory::SourceNatConfig { + ip: ip.ip(), + first_port: *first_port, + last_port: *last_port, + } + } + _ => bail!( + "expected non-NULL snat properties, \ + found at least one NULL" + ), + }; + OmicronZoneType::BoundaryNtp { + address, + dns_servers: ntp_dns_servers?, + domain: self.ntp_domain, + nic: nic?, + ntp_servers: ntp_ntp_servers?, + snat_cfg, + } + } + ZoneType::Clickhouse => { + OmicronZoneType::Clickhouse { address, dataset: dataset? } + } + ZoneType::ClickhouseKeeper => { + OmicronZoneType::ClickhouseKeeper { address, dataset: dataset? } + } + ZoneType::CockroachDb => { + OmicronZoneType::CockroachDb { address, dataset: dataset? } + } + ZoneType::Crucible => { + OmicronZoneType::Crucible { address, dataset: dataset? } + } + ZoneType::CruciblePantry => { + OmicronZoneType::CruciblePantry { address } + } + ZoneType::ExternalDns => OmicronZoneType::ExternalDns { + dataset: dataset?, + dns_address: dns_address?, + http_address: address, + nic: nic?, + }, + ZoneType::InternalDns => OmicronZoneType::InternalDns { + dataset: dataset?, + dns_address: dns_address?, + http_address: address, + gz_address: *self.dns_gz_address.ok_or_else(|| { + anyhow!("expected dns_gz_address, found none") + })?, + gz_address_index: *self.dns_gz_address_index.ok_or_else( + || anyhow!("expected dns_gz_address_index, found none"), + )?, + }, + ZoneType::InternalNtp => OmicronZoneType::InternalNtp { + address, + dns_servers: ntp_dns_servers?, + domain: self.ntp_domain, + ntp_servers: ntp_ntp_servers?, + }, + ZoneType::Nexus => OmicronZoneType::Nexus { + internal_address: address, + nic: nic?, + external_tls: self + .nexus_external_tls + .ok_or_else(|| anyhow!("expected 'external_tls'"))?, + external_ip: self + .second_service_ip + .ok_or_else(|| anyhow!("expected second service IP"))? + .ip(), + external_dns_servers: self + .nexus_external_dns_servers + .ok_or_else(|| anyhow!("expected 'external_dns_servers'"))? + .into_iter() + .map(|i| i.ip()) + .collect(), + }, + ZoneType::Oximeter => OmicronZoneType::Oximeter { address }, + }; + Ok(nexus_types::inventory::OmicronZoneConfig { + id: self.id, + underlay_address: std::net::Ipv6Addr::from(self.underlay_address), + zone_type, + }) + } +} + +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = inv_omicron_zone_nic)] +pub struct InvOmicronZoneNic { + inv_collection_id: Uuid, + pub id: Uuid, + name: Name, + ip: IpNetwork, + mac: MacAddr, + subnet: IpNetwork, + vni: SqlU32, + is_primary: bool, + slot: SqlU8, +} + +impl InvOmicronZoneNic { + pub fn new( + inv_collection_id: Uuid, + zone: &nexus_types::inventory::OmicronZoneConfig, + ) -> Result, anyhow::Error> { + match &zone.zone_type { + OmicronZoneType::ExternalDns { nic, .. } + | OmicronZoneType::BoundaryNtp { nic, .. } + | OmicronZoneType::Nexus { nic, .. } => { + // We do not bother storing the NIC's kind and associated id + // because it should be inferrable from the other information + // that we have. Verify that here. + ensure!( + matches!( + nic.kind, + nexus_types::inventory::NetworkInterfaceKind::Service( + id + ) if id == zone.id + ), + "expected zone's NIC kind to be \"service\" and the \ + id to match the zone's id ({})", + zone.id + ); + + Ok(Some(InvOmicronZoneNic { + inv_collection_id, + id: nic.id, + name: Name::from( + omicron_common::api::external::Name::from( + nic.name.clone(), + ), + ), + ip: IpNetwork::from(nic.ip), + mac: MacAddr::from( + omicron_common::api::external::MacAddr::from( + nic.mac.clone(), + ), + ), + subnet: IpNetwork::from(nic.subnet.clone()), + vni: SqlU32::from(nic.vni.0), + is_primary: nic.primary, + slot: SqlU8::from(nic.slot), + })) + } + _ => Ok(None), + } + } + + pub fn into_network_interface_for_zone( + self, + zone_id: Uuid, + ) -> nexus_types::inventory::NetworkInterface { + nexus_types::inventory::NetworkInterface { + id: self.id, + ip: self.ip.ip(), + kind: nexus_types::inventory::NetworkInterfaceKind::Service( + zone_id, + ), + mac: (*self.mac).into(), + name: (&(*self.name)).into(), + primary: self.is_primary, + slot: *self.slot, + vni: nexus_types::inventory::Vni::from(*self.vni), + subnet: self.subnet.into(), + } + } +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 2c3433b2d3..6b89e5a270 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -35,7 +35,7 @@ mod instance_state; mod inventory; mod ip_pool; mod ipv4net; -mod ipv6; +pub mod ipv6; mod ipv6net; mod l4_port_range; mod macaddr; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 7f4bf51487..791afa6de4 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion; /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(21, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(22, 0, 0); table! { disk (id) { @@ -1331,6 +1331,77 @@ table! { } } +table! { + inv_sled_agent (inv_collection_id, sled_id) { + inv_collection_id -> Uuid, + time_collected -> Timestamptz, + source -> Text, + sled_id -> Uuid, + + hw_baseboard_id -> Nullable, + + sled_agent_ip -> Inet, + sled_agent_port -> Int4, + sled_role -> crate::SledRoleEnum, + usable_hardware_threads -> Int8, + usable_physical_ram -> Int8, + reservoir_size -> Int8, + } +} + +table! { + inv_sled_omicron_zones (inv_collection_id, sled_id) { + inv_collection_id -> Uuid, + time_collected -> Timestamptz, + source -> Text, + sled_id -> Uuid, + + generation -> Int8, + } +} + +table! { + inv_omicron_zone (inv_collection_id, id) { + inv_collection_id -> Uuid, + sled_id -> Uuid, + + id -> Uuid, + underlay_address -> Inet, + zone_type -> crate::ZoneTypeEnum, + + primary_service_ip -> Inet, + primary_service_port -> Int4, + second_service_ip -> Nullable, + second_service_port -> Nullable, + dataset_zpool_name -> Nullable, + nic_id -> Nullable, + dns_gz_address -> Nullable, + dns_gz_address_index -> Nullable, + ntp_ntp_servers -> Nullable>, + ntp_dns_servers -> Nullable>, + ntp_domain -> Nullable, + nexus_external_tls -> Nullable, + nexus_external_dns_servers -> Nullable>, + snat_ip -> Nullable, + snat_first_port -> Nullable, + snat_last_port -> Nullable, + } +} + +table! { + inv_omicron_zone_nic (inv_collection_id, id) { + inv_collection_id -> Uuid, + id -> Uuid, + name -> Text, + ip -> Inet, + mac -> Int8, + subnet -> Inet, + vni -> Int8, + is_primary -> Bool, + slot -> Int2, + } +} + table! { bootstore_keys (key, generation) { key -> Text, @@ -1366,6 +1437,7 @@ allow_tables_to_appear_in_same_query!( sw_root_of_trust_page, inv_root_of_trust_page ); +allow_tables_to_appear_in_same_query!(hw_baseboard_id, inv_sled_agent,); allow_tables_to_appear_in_same_query!( dataset, diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 7d880b4ec0..b7ff058234 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -36,12 +36,20 @@ use nexus_db_model::HwRotSlotEnum; use nexus_db_model::InvCaboose; use nexus_db_model::InvCollection; use nexus_db_model::InvCollectionError; +use nexus_db_model::InvOmicronZone; +use nexus_db_model::InvOmicronZoneNic; use nexus_db_model::InvRootOfTrust; use nexus_db_model::InvRotPage; use nexus_db_model::InvServiceProcessor; +use nexus_db_model::InvSledAgent; +use nexus_db_model::InvSledOmicronZones; use nexus_db_model::RotPageWhichEnum; +use nexus_db_model::SledRole; +use nexus_db_model::SledRoleEnum; use nexus_db_model::SpType; use nexus_db_model::SpTypeEnum; +use nexus_db_model::SqlU16; +use nexus_db_model::SqlU32; use nexus_db_model::SwCaboose; use nexus_db_model::SwRotPage; use nexus_types::inventory::BaseboardId; @@ -108,6 +116,55 @@ impl DataStore { )) }) .collect::, Error>>()?; + // Partition the sled agents into those with an associated baseboard id + // and those without one. We handle these pretty differently. + let (sled_agents_baseboards, sled_agents_no_baseboards): ( + Vec<_>, + Vec<_>, + ) = collection + .sled_agents + .values() + .partition(|sled_agent| sled_agent.baseboard_id.is_some()); + let sled_agents_no_baseboards = sled_agents_no_baseboards + .into_iter() + .map(|sled_agent| { + assert!(sled_agent.baseboard_id.is_none()); + InvSledAgent::new_without_baseboard(collection_id, sled_agent) + .map_err(|e| Error::internal_error(&e.to_string())) + }) + .collect::, Error>>()?; + + let sled_omicron_zones = collection + .omicron_zones + .values() + .map(|found| InvSledOmicronZones::new(collection_id, found)) + .collect::>(); + let omicron_zones = collection + .omicron_zones + .values() + .flat_map(|found| { + found.zones.zones.iter().map(|found_zone| { + InvOmicronZone::new( + collection_id, + found.sled_id, + found_zone, + ) + .map_err(|e| Error::internal_error(&e.to_string())) + }) + }) + .collect::, Error>>()?; + let omicron_zone_nics = collection + .omicron_zones + .values() + .flat_map(|found| { + found.zones.zones.iter().filter_map(|found_zone| { + InvOmicronZoneNic::new(collection_id, found_zone) + .with_context(|| format!("zone {:?}", found_zone.id)) + .map_err(|e| Error::internal_error(&format!("{:#}", e))) + .transpose() + }) + }) + .collect::, _>>()?; // This implementation inserts all records associated with the // collection in one transaction. This is primarily for simplicity. It @@ -573,6 +630,137 @@ impl DataStore { } } + // Insert rows for the sled agents that we found. In practice, we'd + // expect these to all have baseboards (if using Oxide hardware) or + // none have baseboards (if not). + { + use db::schema::hw_baseboard_id::dsl as baseboard_dsl; + use db::schema::inv_sled_agent::dsl as sa_dsl; + + // For sleds with a real baseboard id, we have to use the + // `INSERT INTO ... SELECT` pattern that we used for other types + // of rows above to pull in the baseboard id's uuid. + for sled_agent in &sled_agents_baseboards { + let baseboard_id = sled_agent.baseboard_id.as_ref().expect( + "already selected only sled agents with baseboards", + ); + let selection = db::schema::hw_baseboard_id::table + .select(( + collection_id.into_sql::(), + sled_agent + .time_collected + .into_sql::(), + sled_agent + .source + .clone() + .into_sql::(), + sled_agent + .sled_id + .into_sql::(), + baseboard_dsl::id.nullable(), + nexus_db_model::ipv6::Ipv6Addr::from( + sled_agent.sled_agent_address.ip(), + ) + .into_sql::(), + SqlU16(sled_agent.sled_agent_address.port()) + .into_sql::(), + SledRole::from(sled_agent.sled_role) + .into_sql::(), + SqlU32(sled_agent.usable_hardware_threads) + .into_sql::(), + nexus_db_model::ByteCount::from( + sled_agent.usable_physical_ram, + ) + .into_sql::(), + nexus_db_model::ByteCount::from( + sled_agent.reservoir_size, + ) + .into_sql::(), + )) + .filter( + baseboard_dsl::part_number + .eq(baseboard_id.part_number.clone()), + ) + .filter( + baseboard_dsl::serial_number + .eq(baseboard_id.serial_number.clone()), + ); + + let _ = + diesel::insert_into(db::schema::inv_sled_agent::table) + .values(selection) + .into_columns(( + sa_dsl::inv_collection_id, + sa_dsl::time_collected, + sa_dsl::source, + sa_dsl::sled_id, + sa_dsl::hw_baseboard_id, + sa_dsl::sled_agent_ip, + sa_dsl::sled_agent_port, + sa_dsl::sled_role, + sa_dsl::usable_hardware_threads, + sa_dsl::usable_physical_ram, + sa_dsl::reservoir_size, + )) + .execute_async(&conn) + .await?; + + // See the comment in the earlier block (where we use + // `inv_service_processor::all_columns()`). The same + // applies here. + let ( + _inv_collection_id, + _time_collected, + _source, + _sled_id, + _hw_baseboard_id, + _sled_agent_ip, + _sled_agent_port, + _sled_role, + _usable_hardware_threads, + _usable_physical_ram, + _reservoir_size, + ) = sa_dsl::inv_sled_agent::all_columns(); + } + + // For sleds with no baseboard information, we can't use + // the same INSERT INTO ... SELECT pattern because we + // won't find anything in the hw_baseboard_id table. It + // sucks that these are bifurcated code paths, but on + // the plus side, this is a much simpler INSERT, and we + // can insert all of them in one statement. + let _ = diesel::insert_into(db::schema::inv_sled_agent::table) + .values(sled_agents_no_baseboards) + .execute_async(&conn) + .await?; + } + + // Insert all the Omicron zones that we found. + { + use db::schema::inv_sled_omicron_zones::dsl as sled_zones; + let _ = diesel::insert_into(sled_zones::inv_sled_omicron_zones) + .values(sled_omicron_zones) + .execute_async(&conn) + .await?; + } + + { + use db::schema::inv_omicron_zone::dsl as omicron_zone; + let _ = diesel::insert_into(omicron_zone::inv_omicron_zone) + .values(omicron_zones) + .execute_async(&conn) + .await?; + } + + { + use db::schema::inv_omicron_zone_nic::dsl as omicron_zone_nic; + let _ = + diesel::insert_into(omicron_zone_nic::inv_omicron_zone_nic) + .values(omicron_zone_nics) + .execute_async(&conn) + .await?; + } + // Finally, insert the list of errors. { use db::schema::inv_collection_error::dsl as errors_dsl; @@ -825,7 +1013,18 @@ impl DataStore { // start removing it and we'd also need to make sure we didn't leak a // collection if we crash while deleting it. let conn = self.pool_connection_authorized(opctx).await?; - let (ncollections, nsps, nrots, ncabooses, nrot_pages, nerrors) = conn + let ( + ncollections, + nsps, + nrots, + ncabooses, + nrot_pages, + nsled_agents, + nsled_agent_zones, + nzones, + nnics, + nerrors, + ) = conn .transaction_async(|conn| async move { // Remove the record describing the collection itself. let ncollections = { @@ -881,6 +1080,48 @@ impl DataStore { .await? }; + // Remove rows for sled agents found. + let nsled_agents = { + use db::schema::inv_sled_agent::dsl; + diesel::delete( + dsl::inv_sled_agent + .filter(dsl::inv_collection_id.eq(collection_id)), + ) + .execute_async(&conn) + .await? + }; + + // Remove rows associated with Omicron zones + let nsled_agent_zones = { + use db::schema::inv_sled_omicron_zones::dsl; + diesel::delete( + dsl::inv_sled_omicron_zones + .filter(dsl::inv_collection_id.eq(collection_id)), + ) + .execute_async(&conn) + .await? + }; + + let nzones = { + use db::schema::inv_omicron_zone::dsl; + diesel::delete( + dsl::inv_omicron_zone + .filter(dsl::inv_collection_id.eq(collection_id)), + ) + .execute_async(&conn) + .await? + }; + + let nnics = { + use db::schema::inv_omicron_zone_nic::dsl; + diesel::delete( + dsl::inv_omicron_zone_nic + .filter(dsl::inv_collection_id.eq(collection_id)), + ) + .execute_async(&conn) + .await? + }; + // Remove rows for errors encountered. let nerrors = { use db::schema::inv_collection_error::dsl; @@ -892,7 +1133,18 @@ impl DataStore { .await? }; - Ok((ncollections, nsps, nrots, ncabooses, nrot_pages, nerrors)) + Ok(( + ncollections, + nsps, + nrots, + ncabooses, + nrot_pages, + nsled_agents, + nsled_agent_zones, + nzones, + nnics, + nerrors, + )) }) .await .map_err(|error| match error { @@ -909,6 +1161,10 @@ impl DataStore { "nrots" => nrots, "ncabooses" => ncabooses, "nrot_pages" => nrot_pages, + "nsled_agents" => nsled_agents, + "nsled_agent_zones" => nsled_agent_zones, + "nzones" => nzones, + "nnics" => nnics, "nerrors" => nerrors, ); @@ -1085,9 +1341,27 @@ impl DataStore { }; limit_reached = limit_reached || rots.len() == usize_limit; - // Collect the unique baseboard ids referenced by SPs and RoTs. - let baseboard_id_ids: BTreeSet<_> = - sps.keys().chain(rots.keys()).cloned().collect(); + let sled_agent_rows: Vec<_> = { + use db::schema::inv_sled_agent::dsl; + dsl::inv_sled_agent + .filter(dsl::inv_collection_id.eq(id)) + .limit(sql_limit) + .select(InvSledAgent::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })? + }; + + // Collect the unique baseboard ids referenced by SPs, RoTs, and Sled + // Agents. + let baseboard_id_ids: BTreeSet<_> = sps + .keys() + .chain(rots.keys()) + .cloned() + .chain(sled_agent_rows.iter().filter_map(|s| s.hw_baseboard_id)) + .collect(); // Fetch the corresponding baseboard records. let baseboards_by_id: BTreeMap<_, _> = { use db::schema::hw_baseboard_id::dsl; @@ -1136,6 +1410,49 @@ impl DataStore { }) }) .collect::, _>>()?; + let sled_agents: BTreeMap<_, _> = + sled_agent_rows + .into_iter() + .map(|s: InvSledAgent| { + let sled_id = s.sled_id; + let baseboard_id = s + .hw_baseboard_id + .map(|id| { + baseboards_by_id.get(&id).cloned().ok_or_else( + || { + Error::internal_error( + "missing baseboard that we should have fetched", + ) + }, + ) + }) + .transpose()?; + let sled_agent = nexus_types::inventory::SledAgent { + time_collected: s.time_collected, + source: s.source, + sled_id, + baseboard_id, + sled_agent_address: std::net::SocketAddrV6::new( + std::net::Ipv6Addr::from(s.sled_agent_ip), + u16::from(s.sled_agent_port), + 0, + 0, + ), + sled_role: nexus_types::inventory::SledRole::from( + s.sled_role, + ), + usable_hardware_threads: u32::from( + s.usable_hardware_threads, + ), + usable_physical_ram: s.usable_physical_ram.into(), + reservoir_size: s.reservoir_size.into(), + }; + Ok((sled_id, sled_agent)) + }) + .collect::, + Error, + >>()?; // Fetch records of cabooses found. let inv_caboose_rows = { @@ -1237,7 +1554,7 @@ impl DataStore { .iter() .map(|inv_rot_page| inv_rot_page.sw_root_of_trust_page_id) .collect(); - // Fetch the corresponing records. + // Fetch the corresponding records. let rot_pages_by_id: BTreeMap<_, _> = { use db::schema::sw_root_of_trust_page::dsl; dsl::sw_root_of_trust_page @@ -1299,6 +1616,117 @@ impl DataStore { ); } + // Now read the Omicron zones. + // + // In the first pass, we'll load the "inv_sled_omicron_zones" records. + // There's one of these per sled. It does not contain the actual list + // of zones -- basically just collection metadata and the generation + // number. We'll assemble these directly into the data structure we're + // trying to build, which maps sled ids to objects describing the zones + // found on each sled. + let mut omicron_zones: BTreeMap<_, _> = { + use db::schema::inv_sled_omicron_zones::dsl; + dsl::inv_sled_omicron_zones + .filter(dsl::inv_collection_id.eq(id)) + .limit(sql_limit) + .select(InvSledOmicronZones::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|sled_zones_config| { + ( + sled_zones_config.sled_id, + sled_zones_config.into_uninit_zones_found(), + ) + }) + .collect() + }; + limit_reached = limit_reached || omicron_zones.len() == usize_limit; + + // Assemble a mutable map of all the NICs found, by NIC id. As we + // match these up with the corresponding zone below, we'll remove items + // from this set. That way we can tell if the same NIC was used twice + // or not used at all. + let mut omicron_zone_nics: BTreeMap<_, _> = { + use db::schema::inv_omicron_zone_nic::dsl; + dsl::inv_omicron_zone_nic + .filter(dsl::inv_collection_id.eq(id)) + .limit(sql_limit) + .select(InvOmicronZoneNic::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|found_zone_nic| (found_zone_nic.id, found_zone_nic)) + .collect() + }; + limit_reached = limit_reached || omicron_zone_nics.len() == usize_limit; + + // Now load the actual list of zones from all sleds. + let omicron_zones_list = { + use db::schema::inv_omicron_zone::dsl; + dsl::inv_omicron_zone + .filter(dsl::inv_collection_id.eq(id)) + // It's not strictly necessary to order these by id. Doing so + // ensures a consistent representation for `Collection`, which + // makes testing easier. It's already indexed to do this, too. + .order_by(dsl::id) + .limit(sql_limit) + .select(InvOmicronZone::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })? + }; + limit_reached = + limit_reached || omicron_zones_list.len() == usize_limit; + for z in omicron_zones_list { + let nic_row = z + .nic_id + .map(|id| { + // This error means that we found a row in inv_omicron_zone + // that references a NIC by id but there's no corresponding + // row in inv_omicron_zone_nic with that id. This should be + // impossible and reflects either a bug or database + // corruption. + omicron_zone_nics.remove(&id).ok_or_else(|| { + Error::internal_error(&format!( + "zone {:?}: expected to find NIC {:?}, but didn't", + z.id, z.nic_id + )) + }) + }) + .transpose()?; + let map = omicron_zones.get_mut(&z.sled_id).ok_or_else(|| { + // This error means that we found a row in inv_omicron_zone with + // no associated record in inv_sled_omicron_zones. This should + // be impossible and reflects either a bug or database + // corruption. + Error::internal_error(&format!( + "zone {:?}: unknown sled: {:?}", + z.id, z.sled_id + )) + })?; + let zone_id = z.id; + let zone = z + .into_omicron_zone_config(nic_row) + .with_context(|| { + format!("zone {:?}: parse from database", zone_id) + }) + .map_err(|e| { + Error::internal_error(&format!("{:#}", e.to_string())) + })?; + map.zones.zones.push(zone); + } + + bail_unless!( + omicron_zone_nics.is_empty(), + "found extra Omicron zone NICs: {:?}", + omicron_zone_nics.keys() + ); + Ok(( Collection { id, @@ -1313,6 +1741,8 @@ impl DataStore { rots, cabooses_found, rot_pages_found, + sled_agents, + omicron_zones, }, limit_reached, )) @@ -1476,7 +1906,7 @@ mod test { assert_eq!(collection1, collection_read); // There ought to be no baseboards, cabooses, or RoT pages in the - // databases from that collection. + // database from that collection. assert_eq!(collection1.baseboards.len(), 0); assert_eq!(collection1.cabooses.len(), 0); assert_eq!(collection1.rot_pages.len(), 0); @@ -1815,6 +2245,39 @@ mod test { .await .unwrap(); assert_eq!(0, count); + let count = + schema::inv_root_of_trust_page::dsl::inv_root_of_trust_page + .select(diesel::dsl::count_star()) + .first_async::(&conn) + .await + .unwrap(); + assert_eq!(0, count); + let count = schema::inv_sled_agent::dsl::inv_sled_agent + .select(diesel::dsl::count_star()) + .first_async::(&conn) + .await + .unwrap(); + assert_eq!(0, count); + let count = + schema::inv_sled_omicron_zones::dsl::inv_sled_omicron_zones + .select(diesel::dsl::count_star()) + .first_async::(&conn) + .await + .unwrap(); + assert_eq!(0, count); + let count = schema::inv_omicron_zone::dsl::inv_omicron_zone + .select(diesel::dsl::count_star()) + .first_async::(&conn) + .await + .unwrap(); + assert_eq!(0, count); + let count = schema::inv_omicron_zone_nic::dsl::inv_omicron_zone_nic + .select(diesel::dsl::count_star()) + .first_async::(&conn) + .await + .unwrap(); + assert_eq!(0, count); + Ok::<(), anyhow::Error>(()) }) .await diff --git a/nexus/db-queries/src/db/pool_connection.rs b/nexus/db-queries/src/db/pool_connection.rs index e96a15894d..6fb951de84 100644 --- a/nexus/db-queries/src/db/pool_connection.rs +++ b/nexus/db-queries/src/db/pool_connection.rs @@ -58,6 +58,7 @@ static CUSTOM_TYPE_KEYS: &'static [&'static str] = &[ "service_kind", "sled_provision_state", "sled_resource_kind", + "sled_role", "snapshot_state", "sp_type", "switch_interface_kind", @@ -73,6 +74,7 @@ static CUSTOM_TYPE_KEYS: &'static [&'static str] = &[ "vpc_firewall_rule_protocol", "vpc_firewall_rule_status", "vpc_router_kind", + "zone_type", ]; const CUSTOM_TYPE_SCHEMA: &'static str = "public"; diff --git a/nexus/inventory/Cargo.toml b/nexus/inventory/Cargo.toml index 22b48ebcec..1c20e8f8b6 100644 --- a/nexus/inventory/Cargo.toml +++ b/nexus/inventory/Cargo.toml @@ -8,9 +8,14 @@ license = "MPL-2.0" anyhow.workspace = true base64.workspace = true chrono.workspace = true +futures.workspace = true gateway-client.workspace = true gateway-messages.workspace = true nexus-types.workspace = true +omicron-common.workspace = true +reqwest.workspace = true +serde_json.workspace = true +sled-agent-client.workspace = true slog.workspace = true strum.workspace = true thiserror.workspace = true @@ -20,5 +25,6 @@ omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true gateway-test-utils.workspace = true +omicron-sled-agent.workspace = true regex.workspace = true tokio.workspace = true diff --git a/nexus/inventory/example-data/madrid-sled14.json b/nexus/inventory/example-data/madrid-sled14.json new file mode 100644 index 0000000000..f91c12d3f0 --- /dev/null +++ b/nexus/inventory/example-data/madrid-sled14.json @@ -0,0 +1,214 @@ +{ + "generation": 5, + "zones": [ + { + "id": "0a5f085b-dfb9-4eed-bd24-678bd97e453c", + "underlay_address": "fd00:1122:3344:104::c", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::c]:32345", + "dataset": { + "pool_name": "oxp_e1bf20e5-603c-4d14-94c4-47dc1eb58c45" + } + } + }, + { + "id": "175eb50f-c54c-41ed-b30e-bb710868b362", + "underlay_address": "fd00:1122:3344:104::a", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::a]:32345", + "dataset": { + "pool_name": "oxp_3bcdbecd-827a-426e-96b6-30c355b78301" + } + } + }, + { + "id": "844a964a-831c-4cb9-82b5-3883c9b404db", + "underlay_address": "fd00:1122:3344:104::e", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::e]:32345", + "dataset": { + "pool_name": "oxp_d721dbb5-6a10-4fe8-9d70-fae69ab84676" + } + } + }, + { + "id": "cd8a5031-44a3-4090-86d7-2bfcc3de7942", + "underlay_address": "fd00:1122:3344:104::d", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::d]:32345", + "dataset": { + "pool_name": "oxp_a90de3a7-b760-45b7-ad72-70cd3570a940" + } + } + }, + { + "id": "f7f78c86-f572-49bf-b6cd-24658ddee847", + "underlay_address": "fd00:1122:3344:104::7", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::7]:32345", + "dataset": { + "pool_name": "oxp_5dd3aedf-c3c5-4258-8864-3ea8b5ae321b" + } + } + }, + { + "id": "543e32e4-7d8c-4888-a085-1c530555ee22", + "underlay_address": "fd00:1122:3344:104::6", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::6]:32345", + "dataset": { + "pool_name": "oxp_46b4a891-addc-4690-b8ff-8625b9c5c3bc" + } + } + }, + { + "id": "28786d99-48d2-4491-a4ae-943e603f3dab", + "underlay_address": "fd00:1122:3344:104::9", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::9]:32345", + "dataset": { + "pool_name": "oxp_8c96d804-3a6c-4c1b-be24-1e7fc18824de" + } + } + }, + { + "id": "e59b6fc3-3b0e-4e17-acfa-0351e2924771", + "underlay_address": "fd00:1122:3344:104::b", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::b]:32345", + "dataset": { + "pool_name": "oxp_49154338-0e01-4394-9dd2-cb4c53cbb90f" + } + } + }, + { + "id": "ab67e1fa-337f-45e6-83f0-6e94a9d50fc0", + "underlay_address": "fd00:1122:3344:104::4", + "zone_type": { + "type": "nexus", + "internal_address": "[fd00:1122:3344:104::4]:12221", + "external_ip": "172.20.28.2", + "nic": { + "id": "5d4a7e78-d1e1-41cd-881c-02d808fb90be", + "kind": { + "type": "service", + "id": "ab67e1fa-337f-45e6-83f0-6e94a9d50fc0" + }, + "name": "nexus-ab67e1fa-337f-45e6-83f0-6e94a9d50fc0", + "ip": "172.30.2.5", + "mac": "A8:40:25:FF:B7:E2", + "subnet": "172.30.2.0/24", + "vni": 100, + "primary": true, + "slot": 0 + }, + "external_tls": true, + "external_dns_servers": [ + "1.1.1.1", + "9.9.9.9" + ] + } + }, + { + "id": "5a6d10a6-ce94-444b-82ce-be25ebe58b9a", + "underlay_address": "fd00:1122:3344:104::f", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::f]:32345", + "dataset": { + "pool_name": "oxp_13a6ef76-5904-4794-8083-dfeb6806e5f1" + } + } + }, + { + "id": "442c669b-14d4-48b5-8f05-741b3c67a558", + "underlay_address": "fd00:1122:3344:104::8", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:104::8]:32345", + "dataset": { + "pool_name": "oxp_0010be1f-4223-4f3e-844c-3e823488a852" + } + } + }, + { + "id": "5db69c8f-4565-4cae-8372-f20ada0f67e9", + "underlay_address": "fd00:1122:3344:104::5", + "zone_type": { + "type": "clickhouse", + "address": "[fd00:1122:3344:104::5]:8123", + "dataset": { + "pool_name": "oxp_46b4a891-addc-4690-b8ff-8625b9c5c3bc" + } + } + }, + { + "id": "5d840664-3eb1-45da-8876-d44e1cfb1142", + "underlay_address": "fd00:1122:3344:104::3", + "zone_type": { + "type": "cockroach_db", + "address": "[fd00:1122:3344:104::3]:32221", + "dataset": { + "pool_name": "oxp_46b4a891-addc-4690-b8ff-8625b9c5c3bc" + } + } + }, + { + "id": "d38984ac-a366-4936-b64f-d98ae3dc2035", + "underlay_address": "fd00:1122:3344:104::10", + "zone_type": { + "type": "boundary_ntp", + "address": "[fd00:1122:3344:104::10]:123", + "ntp_servers": [ + "ntp.eng.oxide.computer" + ], + "dns_servers": [ + "1.1.1.1", + "9.9.9.9" + ], + "domain": null, + "nic": { + "id": "2e4943f4-0477-4b5b-afd7-70c1f4aaf928", + "kind": { + "type": "service", + "id": "d38984ac-a366-4936-b64f-d98ae3dc2035" + }, + "name": "ntp-d38984ac-a366-4936-b64f-d98ae3dc2035", + "ip": "172.30.3.6", + "mac": "A8:40:25:FF:C0:38", + "subnet": "172.30.3.0/24", + "vni": 100, + "primary": true, + "slot": 0 + }, + "snat_cfg": { + "ip": "172.20.28.6", + "first_port": 16384, + "last_port": 32767 + } + } + }, + { + "id": "23856e18-8736-49a6-b487-bc5bf850fee0", + "underlay_address": "fd00:1122:3344:2::1", + "zone_type": { + "type": "internal_dns", + "dataset": { + "pool_name": "oxp_46b4a891-addc-4690-b8ff-8625b9c5c3bc" + }, + "http_address": "[fd00:1122:3344:2::1]:5353", + "dns_address": "[fd00:1122:3344:2::1]:53", + "gz_address": "fd00:1122:3344:2::2", + "gz_address_index": 1 + } + } + ] +} diff --git a/nexus/inventory/example-data/madrid-sled16.json b/nexus/inventory/example-data/madrid-sled16.json new file mode 100644 index 0000000000..edf3c71571 --- /dev/null +++ b/nexus/inventory/example-data/madrid-sled16.json @@ -0,0 +1,206 @@ +{ + "generation": 5, + "zones": [ + { + "id": "b2629475-65b2-4e8a-9e70-d4e8c034d8ad", + "underlay_address": "fd00:1122:3344:102::e", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:102::e]:32345", + "dataset": { + "pool_name": "oxp_1cd1c449-b5e1-4e8b-bb2f-2e2bd5a8f301" + } + } + }, + { + "id": "1aa5fd71-d766-4f20-b3c7-9cf4fe9e4f2e", + "underlay_address": "fd00:1122:3344:102::9", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:102::9]:32345", + "dataset": { + "pool_name": "oxp_6d799846-deac-4809-93bd-5dad30127938" + } + } + }, + { + "id": "271ee61b-9e97-4e45-a407-0083f8bf15a7", + "underlay_address": "fd00:1122:3344:102::7", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:102::7]:32345", + "dataset": { + "pool_name": "oxp_65de425b-1487-4d46-85b5-f5fa7c9e776a" + } + } + }, + { + "id": "750b40ef-8e83-4c7a-be96-33964b2244f3", + "underlay_address": "fd00:1122:3344:102::a", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:102::a]:32345", + "dataset": { + "pool_name": "oxp_901a85dd-8214-407a-a358-ef4aebfa810d" + } + } + }, + { + "id": "0322760d-a1e2-4911-8745-569f6bad8251", + "underlay_address": "fd00:1122:3344:102::4", + "zone_type": { + "type": "external_dns", + "dataset": { + "pool_name": "oxp_65de425b-1487-4d46-85b5-f5fa7c9e776a" + }, + "http_address": "[fd00:1122:3344:102::4]:5353", + "dns_address": "172.20.28.1:53", + "nic": { + "id": "8b99b41f-976d-4cb5-bad6-492cde39575a", + "kind": { + "type": "service", + "id": "0322760d-a1e2-4911-8745-569f6bad8251" + }, + "name": "external-dns-0322760d-a1e2-4911-8745-569f6bad8251", + "ip": "172.30.1.5", + "mac": "A8:40:25:FF:F7:4A", + "subnet": "172.30.1.0/24", + "vni": 100, + "primary": true, + "slot": 0 + } + } + }, + { + "id": "f350b534-e9bb-4e47-a2ae-4029efe48e1a", + "underlay_address": "fd00:1122:3344:102::6", + "zone_type": { + "type": "crucible_pantry", + "address": "[fd00:1122:3344:102::6]:17000" + } + }, + { + "id": "e9d7d6ba-59e3-44ff-9081-f43e61c9968a", + "underlay_address": "fd00:1122:3344:102::d", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:102::d]:32345", + "dataset": { + "pool_name": "oxp_51abdeb3-6673-4af3-aa91-7e8748e4dda2" + } + } + }, + { + "id": "d02206f1-7567-4753-9221-6b2b70407925", + "underlay_address": "fd00:1122:3344:102::b", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:102::b]:32345", + "dataset": { + "pool_name": "oxp_0fa59017-d1e7-47c1-9ed6-b66851e544ee" + } + } + }, + { + "id": "c489b9a3-33e5-487c-8a60-77853584dca1", + "underlay_address": "fd00:1122:3344:102::f", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:102::f]:32345", + "dataset": { + "pool_name": "oxp_17d7dbce-b430-4c71-a27e-a5e66d175347" + } + } + }, + { + "id": "996f3011-5aaa-4732-a47d-e6514b1131d8", + "underlay_address": "fd00:1122:3344:102::c", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:102::c]:32345", + "dataset": { + "pool_name": "oxp_87714aed-4573-438c-8c9d-3ed64688bdc4" + } + } + }, + { + "id": "cef138ff-87a4-4509-ba30-2395e01ac5f7", + "underlay_address": "fd00:1122:3344:102::5", + "zone_type": { + "type": "oximeter", + "address": "[fd00:1122:3344:102::5]:12223" + } + }, + { + "id": "263584b3-2f53-4f87-a9c0-60a4c78af6c4", + "underlay_address": "fd00:1122:3344:102::8", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:102::8]:32345", + "dataset": { + "pool_name": "oxp_2acbc210-8b83-490a-b7a7-e458d742c269" + } + } + }, + { + "id": "2f336547-e4b0-422c-af54-deae20b4580c", + "underlay_address": "fd00:1122:3344:102::3", + "zone_type": { + "type": "cockroach_db", + "address": "[fd00:1122:3344:102::3]:32221", + "dataset": { + "pool_name": "oxp_65de425b-1487-4d46-85b5-f5fa7c9e776a" + } + } + }, + { + "id": "412bfd7b-4bf8-471d-ae4d-90bf0bdd05ff", + "underlay_address": "fd00:1122:3344:102::10", + "zone_type": { + "type": "boundary_ntp", + "address": "[fd00:1122:3344:102::10]:123", + "ntp_servers": [ + "ntp.eng.oxide.computer" + ], + "dns_servers": [ + "1.1.1.1", + "9.9.9.9" + ], + "domain": null, + "nic": { + "id": "6c2aa1c5-0e42-4b80-9b31-26d0e8599d0d", + "kind": { + "type": "service", + "id": "412bfd7b-4bf8-471d-ae4d-90bf0bdd05ff" + }, + "name": "ntp-412bfd7b-4bf8-471d-ae4d-90bf0bdd05ff", + "ip": "172.30.3.5", + "mac": "A8:40:25:FF:F2:45", + "subnet": "172.30.3.0/24", + "vni": 100, + "primary": true, + "slot": 0 + }, + "snat_cfg": { + "ip": "172.20.28.5", + "first_port": 0, + "last_port": 16383 + } + } + }, + { + "id": "7de28140-8cdc-4478-9204-63763ecc10ff", + "underlay_address": "fd00:1122:3344:1::1", + "zone_type": { + "type": "internal_dns", + "dataset": { + "pool_name": "oxp_65de425b-1487-4d46-85b5-f5fa7c9e776a" + }, + "http_address": "[fd00:1122:3344:1::1]:5353", + "dns_address": "[fd00:1122:3344:1::1]:53", + "gz_address": "fd00:1122:3344:1::2", + "gz_address_index": 0 + } + } + ] +} diff --git a/nexus/inventory/example-data/madrid-sled17.json b/nexus/inventory/example-data/madrid-sled17.json new file mode 100644 index 0000000000..8ac5dff840 --- /dev/null +++ b/nexus/inventory/example-data/madrid-sled17.json @@ -0,0 +1,172 @@ +{ + "generation": 5, + "zones": [ + { + "id": "e58917eb-98cc-4b85-b851-4b46833060dc", + "underlay_address": "fd00:1122:3344:103::8", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:103::8]:32345", + "dataset": { + "pool_name": "oxp_c6911096-09b3-4f64-bcd6-21701ca2d6ae" + } + } + }, + { + "id": "ae07bfa3-09a9-4b19-9721-c89f39e153fe", + "underlay_address": "fd00:1122:3344:103::7", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:103::7]:32345", + "dataset": { + "pool_name": "oxp_4c667609-0876-4c8c-ae60-1b30aaf236dc" + } + } + }, + { + "id": "6e305032-a926-4c2b-a89a-0165799b9810", + "underlay_address": "fd00:1122:3344:103::9", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:103::9]:32345", + "dataset": { + "pool_name": "oxp_d9974711-1064-441d-ba75-0ffabfc86d27" + } + } + }, + { + "id": "2deb45cb-7160-47fc-9180-ab14c1731427", + "underlay_address": "fd00:1122:3344:103::c", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:103::c]:32345", + "dataset": { + "pool_name": "oxp_e675c45a-5e1b-4d24-99af-806523ed17d5" + } + } + }, + { + "id": "804f9ff7-0d45-465f-8820-ee0fc7c25286", + "underlay_address": "fd00:1122:3344:103::4", + "zone_type": { + "type": "nexus", + "internal_address": "[fd00:1122:3344:103::4]:12221", + "external_ip": "172.20.28.3", + "nic": { + "id": "3e1324f0-cad4-484e-a101-e26da2706e92", + "kind": { + "type": "service", + "id": "804f9ff7-0d45-465f-8820-ee0fc7c25286" + }, + "name": "nexus-804f9ff7-0d45-465f-8820-ee0fc7c25286", + "ip": "172.30.2.6", + "mac": "A8:40:25:FF:A5:50", + "subnet": "172.30.2.0/24", + "vni": 100, + "primary": true, + "slot": 0 + }, + "external_tls": true, + "external_dns_servers": [ + "1.1.1.1", + "9.9.9.9" + ] + } + }, + { + "id": "c28abc48-2fb2-487b-b89a-96317c4e2df2", + "underlay_address": "fd00:1122:3344:103::b", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:103::b]:32345", + "dataset": { + "pool_name": "oxp_93017061-5910-4bf5-a366-4f1b2871b5c3" + } + } + }, + { + "id": "4da2814a-7d31-4311-97cf-7648e7b64911", + "underlay_address": "fd00:1122:3344:103::d", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:103::d]:32345", + "dataset": { + "pool_name": "oxp_33a4881a-2b3f-4840-b252-f370024eee64" + } + } + }, + { + "id": "a3a6216e-fdc8-47b6-8ba8-eb666629f5c2", + "underlay_address": "fd00:1122:3344:103::a", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:103::a]:32345", + "dataset": { + "pool_name": "oxp_844fd687-fd26-4616-91aa-441cf136c62d" + } + } + }, + { + "id": "bd646149-3e59-4aac-b2a0-b79910b8d6a8", + "underlay_address": "fd00:1122:3344:103::6", + "zone_type": { + "type": "crucible", + "address": "[fd00:1122:3344:103::6]:32345", + "dataset": { + "pool_name": "oxp_d130514f-6532-4c02-ac9f-c4958052c669" + } + } + }, + { + "id": "8c9735b2-9097-4ba5-b783-dfea16c5e0ab", + "underlay_address": "fd00:1122:3344:103::5", + "zone_type": { + "type": "crucible_pantry", + "address": "[fd00:1122:3344:103::5]:17000" + } + }, + { + "id": "8fe2fb59-5a89-4d40-b47a-6d5fcfb66ddd", + "underlay_address": "fd00:1122:3344:103::3", + "zone_type": { + "type": "cockroach_db", + "address": "[fd00:1122:3344:103::3]:32221", + "dataset": { + "pool_name": "oxp_d130514f-6532-4c02-ac9f-c4958052c669" + } + } + }, + { + "id": "9ee036cf-88c5-4a0e-aae7-eb2849379aad", + "underlay_address": "fd00:1122:3344:103::e", + "zone_type": { + "type": "internal_ntp", + "address": "[fd00:1122:3344:103::e]:123", + "ntp_servers": [ + "412bfd7b-4bf8-471d-ae4d-90bf0bdd05ff.host.control-plane.oxide.internal", + "d38984ac-a366-4936-b64f-d98ae3dc2035.host.control-plane.oxide.internal" + ], + "dns_servers": [ + "fd00:1122:3344:1::1", + "fd00:1122:3344:2::1", + "fd00:1122:3344:3::1" + ], + "domain": null + } + }, + { + "id": "7b006fad-d693-441b-bdd0-84cb323530e9", + "underlay_address": "fd00:1122:3344:3::1", + "zone_type": { + "type": "internal_dns", + "dataset": { + "pool_name": "oxp_d130514f-6532-4c02-ac9f-c4958052c669" + }, + "http_address": "[fd00:1122:3344:3::1]:5353", + "dns_address": "[fd00:1122:3344:3::1]:53", + "gz_address": "fd00:1122:3344:3::2", + "gz_address_index": 2 + } + } + ] +} diff --git a/nexus/inventory/src/builder.rs b/nexus/inventory/src/builder.rs index 2d8ba0d1f9..602655ef0b 100644 --- a/nexus/inventory/src/builder.rs +++ b/nexus/inventory/src/builder.rs @@ -19,11 +19,14 @@ use nexus_types::inventory::Caboose; use nexus_types::inventory::CabooseFound; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; +use nexus_types::inventory::OmicronZonesFound; use nexus_types::inventory::RotPage; use nexus_types::inventory::RotPageFound; use nexus_types::inventory::RotPageWhich; use nexus_types::inventory::RotState; use nexus_types::inventory::ServiceProcessor; +use nexus_types::inventory::SledAgent; +use omicron_common::api::external::ByteCount; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::sync::Arc; @@ -81,6 +84,8 @@ pub struct CollectionBuilder { BTreeMap, CabooseFound>>, rot_pages_found: BTreeMap, RotPageFound>>, + sleds: BTreeMap, + omicron_zones: BTreeMap, } impl CollectionBuilder { @@ -101,11 +106,19 @@ impl CollectionBuilder { rots: BTreeMap::new(), cabooses_found: BTreeMap::new(), rot_pages_found: BTreeMap::new(), + sleds: BTreeMap::new(), + omicron_zones: BTreeMap::new(), } } /// Assemble a complete `Collection` representation - pub fn build(self) -> Collection { + pub fn build(mut self) -> Collection { + // This is not strictly necessary. But for testing, it's helpful for + // things to be in sorted order. + for v in self.omicron_zones.values_mut() { + v.zones.zones.sort_by(|a, b| a.id.cmp(&b.id)); + } + Collection { id: Uuid::new_v4(), errors: self.errors.into_iter().map(|e| e.to_string()).collect(), @@ -119,6 +132,8 @@ impl CollectionBuilder { rots: self.rots, cabooses_found: self.cabooses_found, rot_pages_found: self.rot_pages_found, + sled_agents: self.sleds, + omicron_zones: self.omicron_zones, } } @@ -387,6 +402,105 @@ impl CollectionBuilder { pub fn found_error(&mut self, error: InventoryError) { self.errors.push(error); } + + /// Record information about a sled that's part of the control plane + pub fn found_sled_inventory( + &mut self, + source: &str, + inventory: sled_agent_client::types::Inventory, + ) -> Result<(), anyhow::Error> { + let sled_id = inventory.sled_id; + + // Normalize the baseboard id, if any. + use sled_agent_client::types::Baseboard; + let baseboard_id = match inventory.baseboard { + Baseboard::Pc { .. } => None, + Baseboard::Gimlet { identifier, model, revision: _ } => { + Some(Self::normalize_item( + &mut self.baseboards, + BaseboardId { + serial_number: identifier, + part_number: model, + }, + )) + } + Baseboard::Unknown => { + self.found_error(InventoryError::from(anyhow!( + "sled {:?}: reported unknown baseboard", + sled_id + ))); + None + } + }; + + // Socket addresses come through the OpenAPI spec as strings, which + // means they don't get validated when everything else does. This + // error is an operational error in collecting the data, not a collector + // bug. + let sled_agent_address = match inventory.sled_agent_address.parse() { + Ok(addr) => addr, + Err(error) => { + self.found_error(InventoryError::from(anyhow!( + "sled {:?}: bad sled agent address: {:?}: {:#}", + sled_id, + inventory.sled_agent_address, + error, + ))); + return Ok(()); + } + }; + let sled = SledAgent { + source: source.to_string(), + sled_agent_address, + sled_role: inventory.sled_role, + baseboard_id, + usable_hardware_threads: inventory.usable_hardware_threads, + usable_physical_ram: ByteCount::from(inventory.usable_physical_ram), + reservoir_size: ByteCount::from(inventory.reservoir_size), + time_collected: now(), + sled_id, + }; + + if let Some(previous) = self.sleds.get(&sled_id) { + Err(anyhow!( + "sled {:?}: reported sled multiple times \ + (previously {:?}, now {:?})", + sled_id, + previous, + sled, + )) + } else { + self.sleds.insert(sled_id, sled); + Ok(()) + } + } + + /// Record information about Omicron zones found on a sled + pub fn found_sled_omicron_zones( + &mut self, + source: &str, + sled_id: Uuid, + zones: sled_agent_client::types::OmicronZonesConfig, + ) -> Result<(), anyhow::Error> { + if let Some(previous) = self.omicron_zones.get(&sled_id) { + Err(anyhow!( + "sled {:?} omicron zones: reported previously: {:?}", + sled_id, + previous + )) + } else { + self.omicron_zones.insert( + sled_id, + OmicronZonesFound { + time_collected: now(), + source: source.to_string(), + sled_id, + zones, + }, + ); + Ok(()) + } + } } /// Returns the current time, truncated to the previous microsecond. @@ -422,6 +536,8 @@ mod test { use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::RotPage; use nexus_types::inventory::RotPageWhich; + use nexus_types::inventory::SledRole; + use omicron_common::api::external::ByteCount; // Verify the contents of an empty collection. #[test] @@ -455,6 +571,8 @@ mod test { // - some missing cabooses // - some cabooses common to multiple baseboards; others not // - serial number reused across different model numbers + // - sled agent inventory + // - omicron zone inventory // // This test is admittedly pretty tedious and maybe not worthwhile but it's // a useful quick check. @@ -463,9 +581,11 @@ mod test { let time_before = now(); let Representative { builder, - sleds: [sled1_bb, sled2_bb, sled3_bb], + sleds: [sled1_bb, sled2_bb, sled3_bb, sled4_bb], switch, psc, + sled_agents: + [sled_agent_id_basic, sled_agent_id_extra, sled_agent_id_pc, sled_agent_id_unknown], } = representative(); let collection = builder.build(); let time_after = now(); @@ -479,21 +599,27 @@ mod test { // no RoT information. assert_eq!( collection.errors.iter().map(|e| e.to_string()).collect::>(), - ["MGS \"fake MGS 1\": reading RoT state for BaseboardId \ + [ + "MGS \"fake MGS 1\": reading RoT state for BaseboardId \ { part_number: \"model1\", serial_number: \"s2\" }: test suite \ - injected error"] + injected error", + "sled 5c5b4cf9-3e13-45fd-871c-f177d6537510: reported unknown \ + baseboard" + ] ); // Verify the baseboard ids found. let expected_baseboards = - &[&sled1_bb, &sled2_bb, &sled3_bb, &switch, &psc]; + &[&sled1_bb, &sled2_bb, &sled3_bb, &sled4_bb, &switch, &psc]; for bb in expected_baseboards { assert!(collection.baseboards.contains(*bb)); } assert_eq!(collection.baseboards.len(), expected_baseboards.len()); // Verify the stuff that's easy to verify for all SPs: timestamps. - assert_eq!(collection.sps.len(), collection.baseboards.len()); + // There will be one more baseboard than SP because of the one added for + // the extra sled agent. + assert_eq!(collection.sps.len() + 1, collection.baseboards.len()); for (bb, sp) in collection.sps.iter() { assert!(collection.time_started <= sp.time_collected); assert!(sp.time_collected <= collection.time_done); @@ -755,6 +881,42 @@ mod test { // plus the common one; same for RoT pages. assert_eq!(collection.cabooses.len(), 5); assert_eq!(collection.rot_pages.len(), 5); + + // Verify that we found the sled agents. + assert_eq!(collection.sled_agents.len(), 4); + for (sled_id, sled_agent) in &collection.sled_agents { + assert_eq!(*sled_id, sled_agent.sled_id); + if *sled_id == sled_agent_id_extra { + assert_eq!(sled_agent.sled_role, SledRole::Scrimlet); + } else { + assert_eq!(sled_agent.sled_role, SledRole::Gimlet); + } + + assert_eq!( + sled_agent.sled_agent_address, + "[::1]:56792".parse().unwrap() + ); + assert_eq!(sled_agent.usable_hardware_threads, 10); + assert_eq!( + sled_agent.usable_physical_ram, + ByteCount::from(1024 * 1024) + ); + assert_eq!(sled_agent.reservoir_size, ByteCount::from(1024)); + } + + let sled1_agent = &collection.sled_agents[&sled_agent_id_basic]; + let sled1_bb = sled1_agent.baseboard_id.as_ref().unwrap(); + assert_eq!(sled1_bb.part_number, "model1"); + assert_eq!(sled1_bb.serial_number, "s1"); + let sled4_agent = &collection.sled_agents[&sled_agent_id_extra]; + let sled4_bb = sled4_agent.baseboard_id.as_ref().unwrap(); + assert_eq!(sled4_bb.serial_number, "s4"); + assert!(collection.sled_agents[&sled_agent_id_pc] + .baseboard_id + .is_none()); + assert!(collection.sled_agents[&sled_agent_id_unknown] + .baseboard_id + .is_none()); } // Exercises all the failure cases that shouldn't happen in real systems. diff --git a/nexus/inventory/src/collector.rs b/nexus/inventory/src/collector.rs index aeca6e43a1..9b335d3ee4 100644 --- a/nexus/inventory/src/collector.rs +++ b/nexus/inventory/src/collector.rs @@ -6,6 +6,7 @@ use crate::builder::CollectionBuilder; use crate::builder::InventoryError; +use crate::SledAgentEnumerator; use anyhow::Context; use gateway_client::types::GetCfpaParams; use gateway_client::types::RotCfpaSlot; @@ -14,25 +15,34 @@ use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; use nexus_types::inventory::RotPage; use nexus_types::inventory::RotPageWhich; +use slog::o; use slog::{debug, error}; use std::sync::Arc; +use std::time::Duration; use strum::IntoEnumIterator; -pub struct Collector { +/// connection and request timeout used for Sled Agent HTTP client +const SLED_AGENT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Collect all inventory data from an Oxide system +pub struct Collector<'a> { log: slog::Logger, mgs_clients: Vec>, + sled_agent_lister: &'a (dyn SledAgentEnumerator + Send + Sync), in_progress: CollectionBuilder, } -impl Collector { +impl<'a> Collector<'a> { pub fn new( creator: &str, mgs_clients: &[Arc], + sled_agent_lister: &'a (dyn SledAgentEnumerator + Send + Sync), log: slog::Logger, ) -> Self { Collector { log, mgs_clients: mgs_clients.to_vec(), + sled_agent_lister, in_progress: CollectionBuilder::new(creator), } } @@ -54,9 +64,8 @@ impl Collector { debug!(&self.log, "begin collection"); - // When we add stages to collect from other components (e.g., sled - // agents), those will go here. self.collect_all_mgs().await; + self.collect_all_sled_agents().await; debug!(&self.log, "finished collection"); @@ -283,15 +292,94 @@ impl Collector { } } } + + /// Collect inventory from all sled agent instances + async fn collect_all_sled_agents(&mut self) { + let urls = match self.sled_agent_lister.list_sled_agents().await { + Err(error) => { + self.in_progress.found_error(error); + return; + } + Ok(clients) => clients, + }; + + for url in urls { + let log = self.log.new(o!("SledAgent" => url.clone())); + let reqwest_client = reqwest::ClientBuilder::new() + .connect_timeout(SLED_AGENT_TIMEOUT) + .timeout(SLED_AGENT_TIMEOUT) + .build() + .unwrap(); + let client = Arc::new(sled_agent_client::Client::new_with_client( + &url, + reqwest_client, + log, + )); + + if let Err(error) = self.collect_one_sled_agent(&client).await { + error!( + &self.log, + "sled agent {:?}: {:#}", + client.baseurl(), + error + ); + } + } + } + + async fn collect_one_sled_agent( + &mut self, + client: &sled_agent_client::Client, + ) -> Result<(), anyhow::Error> { + let sled_agent_url = client.baseurl(); + debug!(&self.log, "begin collection from Sled Agent"; + "sled_agent_url" => client.baseurl() + ); + + let maybe_ident = client.inventory().await.with_context(|| { + format!("Sled Agent {:?}: inventory", &sled_agent_url) + }); + let inventory = match maybe_ident { + Ok(inventory) => inventory.into_inner(), + Err(error) => { + self.in_progress.found_error(InventoryError::from(error)); + return Ok(()); + } + }; + + let sled_id = inventory.sled_id; + self.in_progress.found_sled_inventory(&sled_agent_url, inventory)?; + + let maybe_config = + client.omicron_zones_get().await.with_context(|| { + format!("Sled Agent {:?}: omicron zones", &sled_agent_url) + }); + match maybe_config { + Err(error) => { + self.in_progress.found_error(InventoryError::from(error)); + Ok(()) + } + Ok(zones) => self.in_progress.found_sled_omicron_zones( + &sled_agent_url, + sled_id, + zones.into_inner(), + ), + } + } } #[cfg(test)] mod test { use super::Collector; + use crate::StaticSledAgentEnumerator; use gateway_messages::SpPort; use nexus_types::inventory::Collection; + use omicron_sled_agent::sim; use std::fmt::Write; + use std::net::Ipv6Addr; + use std::net::SocketAddrV6; use std::sync::Arc; + use uuid::Uuid; fn dump_collection(collection: &Collection) -> String { // Construct a stable, human-readable summary of the Collection @@ -379,6 +467,35 @@ mod test { } } + write!(&mut s, "\nsled agents found:\n").unwrap(); + for (sled_id, sled_info) in &collection.sled_agents { + assert_eq!(*sled_id, sled_info.sled_id); + write!(&mut s, " sled {} ({:?})\n", sled_id, sled_info.sled_role) + .unwrap(); + write!(&mut s, " baseboard {:?}\n", sled_info.baseboard_id) + .unwrap(); + + if let Some(found_zones) = collection.omicron_zones.get(sled_id) { + assert_eq!(*sled_id, found_zones.sled_id); + write!( + &mut s, + " zone generation: {:?}\n", + found_zones.zones.generation + ) + .unwrap(); + write!(&mut s, " zones found:\n").unwrap(); + for zone in &found_zones.zones.zones { + write!( + &mut s, + " zone {} type {}\n", + zone.id, + zone.zone_type.label(), + ) + .unwrap(); + } + } + } + write!(&mut s, "\nerrors:\n").unwrap(); for e in &collection.errors { // Some error strings have OS error numbers in them. We want to @@ -402,19 +519,75 @@ mod test { s } + async fn sim_sled_agent( + log: slog::Logger, + sled_id: Uuid, + zone_id: Uuid, + ) -> sim::Server { + // Start a simulated sled agent. + let config = + sim::Config::for_testing(sled_id, sim::SimMode::Auto, None, None); + let agent = sim::Server::start(&config, &log, false).await.unwrap(); + + // Pretend to put some zones onto this sled. We don't need to test this + // exhaustively here because there are builder tests that exercise a + // variety of different data. We just want to make sure that if the + // sled agent reports something specific (some non-degenerate case), + // then it shows up in the resulting collection. + let sled_url = format!("http://{}/", agent.http_server.local_addr()); + let client = sled_agent_client::Client::new(&sled_url, log); + + let zone_address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, 123, 0, 0); + client + .omicron_zones_put(&sled_agent_client::types::OmicronZonesConfig { + generation: sled_agent_client::types::Generation::from(3), + zones: vec![sled_agent_client::types::OmicronZoneConfig { + id: zone_id, + underlay_address: *zone_address.ip(), + zone_type: + sled_agent_client::types::OmicronZoneType::Oximeter { + address: zone_address.to_string(), + }, + }], + }) + .await + .expect("failed to write initial zone version to fake sled agent"); + + agent + } + #[tokio::test] async fn test_basic() { - // Set up the stock MGS test setup which includes a couple of fake SPs. - // Then run a collection against it. + // Set up the stock MGS test setup (which includes a couple of fake SPs) + // and a simulated sled agent. Then run a collection against these. let gwtestctx = gateway_test_utils::setup::test_setup("test_basic", SpPort::One) .await; let log = &gwtestctx.logctx.log; + let sled1 = sim_sled_agent( + log.clone(), + "9cb9b78f-5614-440c-b66d-e8e81fab69b0".parse().unwrap(), + "5125277f-0988-490b-ac01-3bba20cc8f07".parse().unwrap(), + ) + .await; + let sled2 = sim_sled_agent( + log.clone(), + "03265caf-da7d-46c7-b1c2-39fa90ce5c65".parse().unwrap(), + "8b88a56f-3eb6-4d80-ba42-75d867bc427d".parse().unwrap(), + ) + .await; + let sled1_url = format!("http://{}/", sled1.http_server.local_addr()); + let sled2_url = format!("http://{}/", sled2.http_server.local_addr()); let mgs_url = format!("http://{}/", gwtestctx.client.bind_address); let mgs_client = Arc::new(gateway_client::Client::new(&mgs_url, log.clone())); - let collector = - Collector::new("test-suite", &[mgs_client], log.clone()); + let sled_enum = StaticSledAgentEnumerator::new([sled1_url, sled2_url]); + let collector = Collector::new( + "test-suite", + &[mgs_client], + &sled_enum, + log.clone(), + ); let collection = collector .collect_all() .await @@ -425,6 +598,7 @@ mod test { let s = dump_collection(&collection); expectorate::assert_contents("tests/output/collector_basic.txt", &s); + sled1.http_server.close().await.unwrap(); gwtestctx.teardown().await; } @@ -444,6 +618,20 @@ mod test { ) .await; let log = &gwtestctx1.logctx.log; + let sled1 = sim_sled_agent( + log.clone(), + "9cb9b78f-5614-440c-b66d-e8e81fab69b0".parse().unwrap(), + "5125277f-0988-490b-ac01-3bba20cc8f07".parse().unwrap(), + ) + .await; + let sled2 = sim_sled_agent( + log.clone(), + "03265caf-da7d-46c7-b1c2-39fa90ce5c65".parse().unwrap(), + "8b88a56f-3eb6-4d80-ba42-75d867bc427d".parse().unwrap(), + ) + .await; + let sled1_url = format!("http://{}/", sled1.http_server.local_addr()); + let sled2_url = format!("http://{}/", sled2.http_server.local_addr()); let mgs_clients = [&gwtestctx1, &gwtestctx2] .into_iter() .map(|g| { @@ -452,7 +640,9 @@ mod test { Arc::new(client) }) .collect::>(); - let collector = Collector::new("test-suite", &mgs_clients, log.clone()); + let sled_enum = StaticSledAgentEnumerator::new([sled1_url, sled2_url]); + let collector = + Collector::new("test-suite", &mgs_clients, &sled_enum, log.clone()); let collection = collector .collect_all() .await @@ -463,6 +653,7 @@ mod test { let s = dump_collection(&collection); expectorate::assert_contents("tests/output/collector_basic.txt", &s); + sled1.http_server.close().await.unwrap(); gwtestctx1.teardown().await; gwtestctx2.teardown().await; } @@ -490,7 +681,9 @@ mod test { Arc::new(client) }; let mgs_clients = &[bad_client, real_client]; - let collector = Collector::new("test-suite", mgs_clients, log.clone()); + let sled_enum = StaticSledAgentEnumerator::empty(); + let collector = + Collector::new("test-suite", mgs_clients, &sled_enum, log.clone()); let collection = collector .collect_all() .await @@ -502,4 +695,50 @@ mod test { gwtestctx.teardown().await; } + + #[tokio::test] + async fn test_sled_agent_failure() { + // Similar to the basic test, but use multiple sled agents, one of which + // is non-functional. + let gwtestctx = gateway_test_utils::setup::test_setup( + "test_sled_agent_failure", + SpPort::One, + ) + .await; + let log = &gwtestctx.logctx.log; + let sled1 = sim_sled_agent( + log.clone(), + "9cb9b78f-5614-440c-b66d-e8e81fab69b0".parse().unwrap(), + "5125277f-0988-490b-ac01-3bba20cc8f07".parse().unwrap(), + ) + .await; + let sled1_url = format!("http://{}/", sled1.http_server.local_addr()); + let sledbogus_url = String::from("http://[100::1]:45678"); + let mgs_url = format!("http://{}/", gwtestctx.client.bind_address); + let mgs_client = + Arc::new(gateway_client::Client::new(&mgs_url, log.clone())); + let sled_enum = + StaticSledAgentEnumerator::new([sled1_url, sledbogus_url]); + let collector = Collector::new( + "test-suite", + &[mgs_client], + &sled_enum, + log.clone(), + ); + let collection = collector + .collect_all() + .await + .expect("failed to carry out collection"); + assert!(!collection.errors.is_empty()); + assert_eq!(collection.collector, "test-suite"); + + let s = dump_collection(&collection); + expectorate::assert_contents( + "tests/output/collector_sled_agent_errors.txt", + &s, + ); + + sled1.http_server.close().await.unwrap(); + gwtestctx.teardown().await; + } } diff --git a/nexus/inventory/src/examples.rs b/nexus/inventory/src/examples.rs index 0ce3712942..054be457f3 100644 --- a/nexus/inventory/src/examples.rs +++ b/nexus/inventory/src/examples.rs @@ -13,10 +13,12 @@ use gateway_client::types::SpState; use gateway_client::types::SpType; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; +use nexus_types::inventory::OmicronZonesConfig; use nexus_types::inventory::RotPage; use nexus_types::inventory::RotPageWhich; use std::sync::Arc; use strum::IntoEnumIterator; +use uuid::Uuid; /// Returns an example Collection used for testing /// @@ -264,19 +266,136 @@ pub fn representative() -> Representative { // We deliberately provide no RoT pages for sled2. + // Report some sled agents. + // + // This first one will match "sled1_bb"'s baseboard information. + let sled_agent_id_basic = + "c5aec1df-b897-49e4-8085-ccd975f9b529".parse().unwrap(); + builder + .found_sled_inventory( + "fake sled agent 1", + sled_agent( + sled_agent_id_basic, + sled_agent_client::types::Baseboard::Gimlet { + identifier: String::from("s1"), + model: String::from("model1"), + revision: 0, + }, + sled_agent_client::types::SledRole::Gimlet, + ), + ) + .unwrap(); + + // Here, we report a different sled *with* baseboard information that + // doesn't match one of the baseboards we found. This is unlikely but could + // happen. Make this one a Scrimlet. + let sled4_bb = Arc::new(BaseboardId { + part_number: String::from("model1"), + serial_number: String::from("s4"), + }); + let sled_agent_id_extra = + "d7efa9c4-833d-4354-a9a2-94ba9715c154".parse().unwrap(); + builder + .found_sled_inventory( + "fake sled agent 4", + sled_agent( + sled_agent_id_extra, + sled_agent_client::types::Baseboard::Gimlet { + identifier: sled4_bb.serial_number.clone(), + model: sled4_bb.part_number.clone(), + revision: 0, + }, + sled_agent_client::types::SledRole::Scrimlet, + ), + ) + .unwrap(); + + // Now report a different sled as though it were a PC. It'd be unlikely to + // see a mix of real Oxide hardware and PCs in the same deployment, but this + // exercises different code paths. + let sled_agent_id_pc = + "c4a5325b-e852-4747-b28a-8aaa7eded8a0".parse().unwrap(); + builder + .found_sled_inventory( + "fake sled agent 5", + sled_agent( + sled_agent_id_pc, + sled_agent_client::types::Baseboard::Pc { + identifier: String::from("fellofftruck1"), + model: String::from("fellofftruck"), + }, + sled_agent_client::types::SledRole::Gimlet, + ), + ) + .unwrap(); + + // Finally, report a sled with unknown baseboard information. This should + // look the same as the PC as far as inventory is concerned but let's verify + // it. + let sled_agent_id_unknown = + "5c5b4cf9-3e13-45fd-871c-f177d6537510".parse().unwrap(); + + builder + .found_sled_inventory( + "fake sled agent 6", + sled_agent( + sled_agent_id_unknown, + sled_agent_client::types::Baseboard::Unknown, + sled_agent_client::types::SledRole::Gimlet, + ), + ) + .unwrap(); + + // Report a representative set of Omicron zones. + // + // We've hand-selected a minimal set of files to cover each type of zone. + // These files were constructed by: + // + // (1) copying the "omicron zones" ledgers from the sleds in a working + // Omicron deployment + // (2) pretty-printing each one with `json --in-place --file FILENAME` + // (3) adjusting the format slightly with + // `jq '{ generation: .omicron_generation, zones: .zones }'` + let sled14_data = include_str!("../example-data/madrid-sled14.json"); + let sled16_data = include_str!("../example-data/madrid-sled16.json"); + let sled17_data = include_str!("../example-data/madrid-sled17.json"); + let sled14: OmicronZonesConfig = serde_json::from_str(sled14_data).unwrap(); + let sled16: OmicronZonesConfig = serde_json::from_str(sled16_data).unwrap(); + let sled17: OmicronZonesConfig = serde_json::from_str(sled17_data).unwrap(); + + let sled14_id = "7612d745-d978-41c8-8ee0-84564debe1d2".parse().unwrap(); + builder + .found_sled_omicron_zones("fake sled 14 agent", sled14_id, sled14) + .unwrap(); + let sled16_id = "af56cb43-3422-4f76-85bf-3f229db5f39c".parse().unwrap(); + builder + .found_sled_omicron_zones("fake sled 15 agent", sled16_id, sled16) + .unwrap(); + let sled17_id = "6eb2a0d9-285d-4e03-afa1-090e4656314b".parse().unwrap(); + builder + .found_sled_omicron_zones("fake sled 15 agent", sled17_id, sled17) + .unwrap(); + Representative { builder, - sleds: [sled1_bb, sled2_bb, sled3_bb], + sleds: [sled1_bb, sled2_bb, sled3_bb, sled4_bb], switch: switch1_bb, psc: psc_bb, + sled_agents: [ + sled_agent_id_basic, + sled_agent_id_extra, + sled_agent_id_pc, + sled_agent_id_unknown, + ], } } pub struct Representative { pub builder: CollectionBuilder, - pub sleds: [Arc; 3], + pub sleds: [Arc; 4], pub switch: Arc, pub psc: Arc, + pub sled_agents: [Uuid; 4], } /// Returns an SP state that can be used to populate a collection for testing @@ -314,3 +433,21 @@ pub fn rot_page(unique: &str) -> RotPage { data_base64: base64::engine::general_purpose::STANDARD.encode(unique), } } + +pub fn sled_agent( + sled_id: Uuid, + baseboard: sled_agent_client::types::Baseboard, + sled_role: sled_agent_client::types::SledRole, +) -> sled_agent_client::types::Inventory { + sled_agent_client::types::Inventory { + baseboard, + reservoir_size: sled_agent_client::types::ByteCount::from(1024), + sled_role, + sled_agent_address: "[::1]:56792".parse().unwrap(), + sled_id, + usable_hardware_threads: 10, + usable_physical_ram: sled_agent_client::types::ByteCount::from( + 1024 * 1024, + ), + } +} diff --git a/nexus/inventory/src/lib.rs b/nexus/inventory/src/lib.rs index e92c46916d..f11af8fede 100644 --- a/nexus/inventory/src/lib.rs +++ b/nexus/inventory/src/lib.rs @@ -20,6 +20,7 @@ mod builder; mod collector; pub mod examples; +mod sled_agent_enumerator; // only exposed for test code to construct collections pub use builder::CollectionBuilder; @@ -27,3 +28,6 @@ pub use builder::CollectorBug; pub use builder::InventoryError; pub use collector::Collector; + +pub use sled_agent_enumerator::SledAgentEnumerator; +pub use sled_agent_enumerator::StaticSledAgentEnumerator; diff --git a/nexus/inventory/src/sled_agent_enumerator.rs b/nexus/inventory/src/sled_agent_enumerator.rs new file mode 100644 index 0000000000..8a1b480e3f --- /dev/null +++ b/nexus/inventory/src/sled_agent_enumerator.rs @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::InventoryError; +use futures::future::BoxFuture; +use futures::FutureExt; + +/// Describes how to find the list of sled agents to collect from +/// +/// In a real system, this queries the database to list all sleds. But for +/// testing the `StaticSledAgentEnumerator` below can be used to avoid a +/// database dependency. +pub trait SledAgentEnumerator { + /// Returns a list of URLs for Sled Agent HTTP endpoints + fn list_sled_agents( + &self, + ) -> BoxFuture<'_, Result, InventoryError>>; +} + +/// Used to provide an explicit list of sled agents to a `Collector` +/// +/// This is mainly used for testing. +pub struct StaticSledAgentEnumerator { + agents: Vec, +} + +impl StaticSledAgentEnumerator { + pub fn new(iter: impl IntoIterator) -> Self { + StaticSledAgentEnumerator { agents: iter.into_iter().collect() } + } + + pub fn empty() -> Self { + Self::new(std::iter::empty()) + } +} + +impl SledAgentEnumerator for StaticSledAgentEnumerator { + fn list_sled_agents( + &self, + ) -> BoxFuture<'_, Result, InventoryError>> { + futures::future::ready(Ok(self.agents.clone())).boxed() + } +} diff --git a/nexus/inventory/tests/output/collector_basic.txt b/nexus/inventory/tests/output/collector_basic.txt index b9894ff184..e59e19967a 100644 --- a/nexus/inventory/tests/output/collector_basic.txt +++ b/nexus/inventory/tests/output/collector_basic.txt @@ -3,6 +3,8 @@ baseboards: part "FAKE_SIM_GIMLET" serial "SimGimlet01" part "FAKE_SIM_SIDECAR" serial "SimSidecar0" part "FAKE_SIM_SIDECAR" serial "SimSidecar1" + part "sim-gimlet" serial "sim-03265caf-da7d-46c7-b1c2-39fa90ce5c65" + part "sim-gimlet" serial "sim-9cb9b78f-5614-440c-b66d-e8e81fab69b0" cabooses: board "SimGimletSp" name "SimGimlet" version "0.0.1" git_commit "ffffffff" @@ -68,4 +70,16 @@ rot pages found: CfpaScratch baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": data_base64 "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" CfpaScratch baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": data_base64 "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +sled agents found: + sled 03265caf-da7d-46c7-b1c2-39fa90ce5c65 (Gimlet) + baseboard Some(BaseboardId { part_number: "sim-gimlet", serial_number: "sim-03265caf-da7d-46c7-b1c2-39fa90ce5c65" }) + zone generation: Generation(3) + zones found: + zone 8b88a56f-3eb6-4d80-ba42-75d867bc427d type oximeter + sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Gimlet) + baseboard Some(BaseboardId { part_number: "sim-gimlet", serial_number: "sim-9cb9b78f-5614-440c-b66d-e8e81fab69b0" }) + zone generation: Generation(3) + zones found: + zone 5125277f-0988-490b-ac01-3bba20cc8f07 type oximeter + errors: diff --git a/nexus/inventory/tests/output/collector_errors.txt b/nexus/inventory/tests/output/collector_errors.txt index a50e24ca30..c39d6b249a 100644 --- a/nexus/inventory/tests/output/collector_errors.txt +++ b/nexus/inventory/tests/output/collector_errors.txt @@ -68,5 +68,7 @@ rot pages found: CfpaScratch baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": data_base64 "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" CfpaScratch baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": data_base64 "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +sled agents found: + errors: error: MGS "http://[100::1]:12345": listing ignition targets: Communication Error <> diff --git a/nexus/inventory/tests/output/collector_sled_agent_errors.txt b/nexus/inventory/tests/output/collector_sled_agent_errors.txt new file mode 100644 index 0000000000..9ebf2cece9 --- /dev/null +++ b/nexus/inventory/tests/output/collector_sled_agent_errors.txt @@ -0,0 +1,80 @@ +baseboards: + part "FAKE_SIM_GIMLET" serial "SimGimlet00" + part "FAKE_SIM_GIMLET" serial "SimGimlet01" + part "FAKE_SIM_SIDECAR" serial "SimSidecar0" + part "FAKE_SIM_SIDECAR" serial "SimSidecar1" + part "sim-gimlet" serial "sim-9cb9b78f-5614-440c-b66d-e8e81fab69b0" + +cabooses: + board "SimGimletSp" name "SimGimlet" version "0.0.1" git_commit "ffffffff" + board "SimRot" name "SimGimlet" version "0.0.1" git_commit "eeeeeeee" + board "SimRot" name "SimSidecar" version "0.0.1" git_commit "eeeeeeee" + board "SimSidecarSp" name "SimSidecar" version "0.0.1" git_commit "ffffffff" + +rot pages: + data_base64 "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + data_base64 "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + data_base64 "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + data_base64 "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + data_base64 "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + data_base64 "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + data_base64 "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + data_base64 "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + +SPs: + baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00" + baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01" + baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0" + baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1" + +RoTs: + baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00" + baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01" + baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0" + baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1" + +cabooses found: + SpSlot0 baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00": board "SimGimletSp" + SpSlot0 baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01": board "SimGimletSp" + SpSlot0 baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": board "SimSidecarSp" + SpSlot0 baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": board "SimSidecarSp" + SpSlot1 baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00": board "SimGimletSp" + SpSlot1 baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01": board "SimGimletSp" + SpSlot1 baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": board "SimSidecarSp" + SpSlot1 baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": board "SimSidecarSp" + RotSlotA baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00": board "SimRot" + RotSlotA baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01": board "SimRot" + RotSlotA baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": board "SimRot" + RotSlotA baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": board "SimRot" + RotSlotB baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00": board "SimRot" + RotSlotB baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01": board "SimRot" + RotSlotB baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": board "SimRot" + RotSlotB baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": board "SimRot" + +rot pages found: + Cmpa baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00": data_base64 "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + Cmpa baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01": data_base64 "Z2ltbGV0LWNtcGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + Cmpa baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": data_base64 "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + Cmpa baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": data_base64 "c2lkZWNhci1jbXBhAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaActive baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00": data_base64 "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaActive baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01": data_base64 "Z2ltbGV0LWNmcGEtYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaActive baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": data_base64 "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaActive baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": data_base64 "c2lkZWNhci1jZnBhLWFjdGl2ZQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaInactive baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00": data_base64 "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaInactive baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01": data_base64 "Z2ltbGV0LWNmcGEtaW5hY3RpdmUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaInactive baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": data_base64 "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaInactive baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": data_base64 "c2lkZWNhci1jZnBhLWluYWN0aXZlAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaScratch baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet00": data_base64 "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaScratch baseboard part "FAKE_SIM_GIMLET" serial "SimGimlet01": data_base64 "Z2ltbGV0LWNmcGEtc2NyYXRjaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaScratch baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar0": data_base64 "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + CfpaScratch baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": data_base64 "c2lkZWNhci1jZnBhLXNjcmF0Y2gAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + +sled agents found: + sled 9cb9b78f-5614-440c-b66d-e8e81fab69b0 (Gimlet) + baseboard Some(BaseboardId { part_number: "sim-gimlet", serial_number: "sim-9cb9b78f-5614-440c-b66d-e8e81fab69b0" }) + zone generation: Generation(3) + zones found: + zone 5125277f-0988-490b-ac01-3bba20cc8f07 type oximeter + +errors: +error: Sled Agent "http://[100::1]:45678": inventory: Communication Error <> diff --git a/nexus/src/app/background/inventory_collection.rs b/nexus/src/app/background/inventory_collection.rs index f095b094db..5c52fa519b 100644 --- a/nexus/src/app/background/inventory_collection.rs +++ b/nexus/src/app/background/inventory_collection.rs @@ -11,11 +11,18 @@ use futures::future::BoxFuture; use futures::FutureExt; use internal_dns::ServiceName; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::pagination::Paginator; use nexus_db_queries::db::DataStore; +use nexus_inventory::InventoryError; +use nexus_types::identity::Asset; use nexus_types::inventory::Collection; use serde_json::json; +use std::num::NonZeroU32; use std::sync::Arc; +/// How many rows to request in each paginated database query +const DB_PAGE_SIZE: u32 = 1024; + /// Background task that reads inventory for the rack pub struct InventoryCollector { datastore: Arc, @@ -123,10 +130,15 @@ async fn inventory_activate( }) .collect::>(); + // Create an enumerator to find sled agents. + let page_size = NonZeroU32::new(DB_PAGE_SIZE).unwrap(); + let sled_enum = DbSledAgentEnumerator { opctx, datastore, page_size }; + // Run a collection. let inventory = nexus_inventory::Collector::new( creator, &mgs_clients, + &sled_enum, opctx.log.clone(), ); let collection = @@ -141,14 +153,64 @@ async fn inventory_activate( Ok(collection) } +/// Determine which sleds to inventory based on what's in the database +/// +/// We only want to inventory what's actually part of the control plane (i.e., +/// has a "sled" record). +struct DbSledAgentEnumerator<'a> { + opctx: &'a OpContext, + datastore: &'a DataStore, + page_size: NonZeroU32, +} + +impl<'a> nexus_inventory::SledAgentEnumerator for DbSledAgentEnumerator<'a> { + fn list_sled_agents( + &self, + ) -> BoxFuture<'_, Result, InventoryError>> { + async { + let mut all_sleds = Vec::new(); + let mut paginator = Paginator::new(self.page_size); + while let Some(p) = paginator.next() { + let records_batch = self + .datastore + .sled_list(&self.opctx, &p.current_pagparams()) + .await + .context("listing sleds")?; + paginator = p.found_batch( + &records_batch, + &|s: &nexus_db_model::Sled| s.id(), + ); + all_sleds.extend( + records_batch + .into_iter() + .map(|sled| format!("http://{}", sled.address())), + ); + } + + Ok(all_sleds) + } + .boxed() + } +} + #[cfg(test)] mod test { use crate::app::background::common::BackgroundTask; + use crate::app::background::inventory_collection::DbSledAgentEnumerator; use crate::app::background::inventory_collection::InventoryCollector; + use nexus_db_model::SledBaseboard; + use nexus_db_model::SledSystemHardware; + use nexus_db_model::SledUpdate; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::DataStoreInventoryTest; + use nexus_inventory::SledAgentEnumerator; use nexus_test_utils_macros::nexus_test; + use omicron_common::api::external::ByteCount; use omicron_test_utils::dev::poll; + use std::net::Ipv6Addr; + use std::net::SocketAddrV6; + use std::num::NonZeroU32; + use uuid::Uuid; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -240,4 +302,80 @@ mod test { let latest = datastore.inventory_collections().await.unwrap(); assert_eq!(previous, latest); } + + #[nexus_test(server = crate::Server)] + async fn test_db_sled_enumerator(cptestctx: &ControlPlaneTestContext) { + let nexus = &cptestctx.server.apictx().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + let db_enum = DbSledAgentEnumerator { + opctx: &opctx, + datastore: &datastore, + page_size: NonZeroU32::new(3).unwrap(), + }; + + // There will be one sled agent set up as part of the test context. + let found_urls = db_enum.list_sled_agents().await.unwrap(); + assert_eq!(found_urls.len(), 1); + + // Insert some sleds. + let rack_id = Uuid::new_v4(); + let mut sleds = Vec::new(); + for i in 0..64 { + let sled = SledUpdate::new( + Uuid::new_v4(), + SocketAddrV6::new(Ipv6Addr::LOCALHOST, 1200 + i, 0, 0), + SledBaseboard { + serial_number: format!("serial-{}", i), + part_number: String::from("fake-sled"), + revision: 3, + }, + SledSystemHardware { + is_scrimlet: false, + usable_hardware_threads: 12, + usable_physical_ram: ByteCount::from_gibibytes_u32(16) + .into(), + reservoir_size: ByteCount::from_gibibytes_u32(8).into(), + }, + rack_id, + ); + sleds.push(datastore.sled_upsert(sled).await.unwrap()); + } + + // The same enumerator should immediately find all the new sleds. + let mut expected_urls: Vec<_> = found_urls + .into_iter() + .chain(sleds.into_iter().map(|s| format!("http://{}", s.address()))) + .collect(); + expected_urls.sort(); + println!("expected_urls: {:?}", expected_urls); + + let mut found_urls = db_enum.list_sled_agents().await.unwrap(); + found_urls.sort(); + assert_eq!(expected_urls, found_urls); + + // We should get the same result even with a page size of 1. + let db_enum = DbSledAgentEnumerator { + opctx: &opctx, + datastore: &datastore, + page_size: NonZeroU32::new(1).unwrap(), + }; + let mut found_urls = db_enum.list_sled_agents().await.unwrap(); + found_urls.sort(); + assert_eq!(expected_urls, found_urls); + + // We should get the same result even with a page size much larger than + // we need. + let db_enum = DbSledAgentEnumerator { + opctx: &opctx, + datastore: &datastore, + page_size: NonZeroU32::new(1024).unwrap(), + }; + let mut found_urls = db_enum.list_sled_agents().await.unwrap(); + found_urls.sort(); + assert_eq!(expected_urls, found_urls); + } } diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index 44efc2934e..943490ac04 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -96,9 +96,9 @@ impl super::Nexus { // but for now, connections to sled agents are constructed // on an "as requested" basis. // - // Franky, returning an "Arc" here without a connection pool is a little - // silly; it's not actually used if each client connection exists as a - // one-shot. + // Frankly, returning an "Arc" here without a connection pool is a + // little silly; it's not actually used if each client connection exists + // as a one-shot. let (.., sled) = self.sled_lookup(&self.opctx_alloc, id)?.fetch().await?; diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 52ff8910f9..d2ac0405fc 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -51,6 +51,9 @@ use trust_dns_resolver::config::ResolverOpts; use trust_dns_resolver::TokioAsyncResolver; use uuid::Uuid; +pub use sim::TEST_HARDWARE_THREADS; +pub use sim::TEST_RESERVOIR_RAM; + pub mod db; pub mod http_testing; pub mod resource_helpers; @@ -62,13 +65,6 @@ pub const OXIMETER_UUID: &str = "39e6175b-4df2-4730-b11d-cbc1e60a2e78"; pub const PRODUCER_UUID: &str = "a6458b7d-87c3-4483-be96-854d814c20de"; pub const RACK_SUBNET: &str = "fd00:1122:3344:01::/56"; -/// The reported amount of hardware threads for an emulated sled agent. -pub const TEST_HARDWARE_THREADS: u32 = 16; -/// The reported amount of physical RAM for an emulated sled agent. -pub const TEST_PHYSICAL_RAM: u64 = 32 * (1 << 30); -/// The reported amount of VMM reservoir RAM for an emulated sled agent. -pub const TEST_RESERVOIR_RAM: u64 = 16 * (1 << 30); - /// Password for the user created by the test suite /// /// This is only used by the test suite and `omicron-dev run-all` (the latter of @@ -994,32 +990,15 @@ pub async fn start_sled_agent( update_directory: &Utf8Path, sim_mode: sim::SimMode, ) -> Result { - let config = sim::Config { + let config = sim::Config::for_testing( id, sim_mode, - nexus_address, - dropshot: ConfigDropshot { - bind_address: SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 0), - request_body_max_bytes: 1024 * 1024, - default_handler_task_mode: HandlerTaskMode::Detached, - }, - // TODO-cleanup this is unused - log: ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Debug }, - storage: sim::ConfigStorage { - zpools: vec![], - ip: IpAddr::from(Ipv6Addr::LOCALHOST), - }, - updates: sim::ConfigUpdates { - zone_artifact_path: update_directory.to_path_buf(), - }, - hardware: sim::ConfigHardware { - hardware_threads: TEST_HARDWARE_THREADS, - physical_ram: TEST_PHYSICAL_RAM, - reservoir_ram: TEST_RESERVOIR_RAM, - }, - }; - let server = - sim::Server::start(&config, &log).await.map_err(|e| e.to_string())?; + Some(nexus_address), + Some(update_directory), + ); + let server = sim::Server::start(&config, &log, true) + .await + .map_err(|e| e.to_string())?; Ok(server) } diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index 9cb94a8484..90ec67c0e6 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -24,3 +24,4 @@ gateway-client.workspace = true omicron-common.workspace = true omicron-passwords.workspace = true omicron-workspace-hack.workspace = true +sled-agent-client.workspace = true diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 77bc73306d..b27d7277ba 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -9,20 +9,31 @@ //! nexus/inventory does not currently know about nexus/db-model and it's //! convenient to separate these concerns.) +use crate::external_api::params::UninitializedSledId; +use crate::external_api::shared::Baseboard; use chrono::DateTime; use chrono::Utc; pub use gateway_client::types::PowerState; pub use gateway_client::types::RotSlot; pub use gateway_client::types::SpType; +use omicron_common::api::external::ByteCount; +pub use sled_agent_client::types::NetworkInterface; +pub use sled_agent_client::types::NetworkInterfaceKind; +pub use sled_agent_client::types::OmicronZoneConfig; +pub use sled_agent_client::types::OmicronZoneDataset; +pub use sled_agent_client::types::OmicronZoneType; +pub use sled_agent_client::types::OmicronZonesConfig; +pub use sled_agent_client::types::SledRole; +pub use sled_agent_client::types::SourceNatConfig; +pub use sled_agent_client::types::Vni; +pub use sled_agent_client::types::ZpoolName; use std::collections::BTreeMap; use std::collections::BTreeSet; +use std::net::SocketAddrV6; use std::sync::Arc; use strum::EnumIter; use uuid::Uuid; -use crate::external_api::params::UninitializedSledId; -use crate::external_api::shared::Baseboard; - /// Results of collecting hardware/software inventory from various Omicron /// components /// @@ -89,6 +100,12 @@ pub struct Collection { /// table. pub rot_pages_found: BTreeMap, RotPageFound>>, + + /// Sled Agent information, by *sled* id + pub sled_agents: BTreeMap, + + /// Omicron zones found, by *sled* id + pub omicron_zones: BTreeMap, } impl Collection { @@ -269,3 +286,30 @@ impl IntoRotPage for gateway_client::types::RotCfpa { (which, RotPage { data_base64: self.base64_data }) } } + +/// Inventory reported by sled agent +/// +/// This is a software notion of a sled, distinct from an underlying baseboard. +/// A sled may be on a PC (in dev/test environments) and have no associated +/// baseboard. There might also be baseboards with no associated sled (if +/// they have not been formally added to the control plane). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SledAgent { + pub time_collected: DateTime, + pub source: String, + pub sled_id: Uuid, + pub baseboard_id: Option>, + pub sled_agent_address: SocketAddrV6, + pub sled_role: SledRole, + pub usable_hardware_threads: u32, + pub usable_physical_ram: ByteCount, + pub reservoir_size: ByteCount, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OmicronZonesFound { + pub time_collected: DateTime, + pub source: String, + pub sled_id: Uuid, + pub zones: OmicronZonesConfig, +} diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 467fd32cb8..b5b9d3fd5b 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -415,6 +415,30 @@ } } }, + "/inventory": { + "get": { + "summary": "Fetch basic information about this sled", + "operationId": "inventory", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Inventory" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/metrics/collect/{producer_id}": { "get": { "summary": "Collect oximeter samples from the sled agent.", @@ -4916,6 +4940,45 @@ } } }, + "Inventory": { + "description": "Identity and basic status information about this sled agent", + "type": "object", + "properties": { + "baseboard": { + "$ref": "#/components/schemas/Baseboard" + }, + "reservoir_size": { + "$ref": "#/components/schemas/ByteCount" + }, + "sled_agent_address": { + "type": "string" + }, + "sled_id": { + "type": "string", + "format": "uuid" + }, + "sled_role": { + "$ref": "#/components/schemas/SledRole" + }, + "usable_hardware_threads": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "usable_physical_ram": { + "$ref": "#/components/schemas/ByteCount" + } + }, + "required": [ + "baseboard", + "reservoir_size", + "sled_agent_address", + "sled_id", + "sled_role", + "usable_hardware_threads", + "usable_physical_ram" + ] + }, "IpNet": { "oneOf": [ { @@ -6154,6 +6217,7 @@ ] }, "SledRole": { + "description": "Describes the role of the sled within the rack.\n\nNote that this may change if the sled is physically moved within the rack.", "oneOf": [ { "description": "The sled is a general compute sled.", diff --git a/schema/crdb/22.0.0/up01.sql b/schema/crdb/22.0.0/up01.sql new file mode 100644 index 0000000000..2e7699d24b --- /dev/null +++ b/schema/crdb/22.0.0/up01.sql @@ -0,0 +1,4 @@ +CREATE TYPE IF NOT EXISTS omicron.public.sled_role AS ENUM ( + 'scrimlet', + 'gimlet' +); diff --git a/schema/crdb/22.0.0/up02.sql b/schema/crdb/22.0.0/up02.sql new file mode 100644 index 0000000000..8f8ddea015 --- /dev/null +++ b/schema/crdb/22.0.0/up02.sql @@ -0,0 +1,19 @@ +CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_agent ( + inv_collection_id UUID NOT NULL, + time_collected TIMESTAMPTZ NOT NULL, + source TEXT NOT NULL, + + sled_id UUID NOT NULL, + + hw_baseboard_id UUID, + + sled_agent_ip INET NOT NULL, + sled_agent_port INT4 NOT NULL, + sled_role omicron.public.sled_role NOT NULL, + usable_hardware_threads INT8 + CHECK (usable_hardware_threads BETWEEN 0 AND 4294967295) NOT NULL, + usable_physical_ram INT8 NOT NULL, + reservoir_size INT8 CHECK (reservoir_size < usable_physical_ram) NOT NULL, + + PRIMARY KEY (inv_collection_id, sled_id) +); diff --git a/schema/crdb/22.0.0/up03.sql b/schema/crdb/22.0.0/up03.sql new file mode 100644 index 0000000000..b741141b2b --- /dev/null +++ b/schema/crdb/22.0.0/up03.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_omicron_zones ( + inv_collection_id UUID NOT NULL, + time_collected TIMESTAMPTZ NOT NULL, + source TEXT NOT NULL, + + sled_id UUID NOT NULL, + + generation INT8 NOT NULL, + + PRIMARY KEY (inv_collection_id, sled_id) +); diff --git a/schema/crdb/22.0.0/up04.sql b/schema/crdb/22.0.0/up04.sql new file mode 100644 index 0000000000..74620e9685 --- /dev/null +++ b/schema/crdb/22.0.0/up04.sql @@ -0,0 +1,13 @@ +CREATE TYPE IF NOT EXISTS omicron.public.zone_type AS ENUM ( + 'boundary_ntp', + 'clickhouse', + 'clickhouse_keeper', + 'cockroach_db', + 'crucible', + 'crucible_pantry', + 'external_dns', + 'internal_dns', + 'internal_ntp', + 'nexus', + 'oximeter' +); diff --git a/schema/crdb/22.0.0/up05.sql b/schema/crdb/22.0.0/up05.sql new file mode 100644 index 0000000000..11d8684854 --- /dev/null +++ b/schema/crdb/22.0.0/up05.sql @@ -0,0 +1,41 @@ +CREATE TABLE IF NOT EXISTS omicron.public.inv_omicron_zone ( + inv_collection_id UUID NOT NULL, + + sled_id UUID NOT NULL, + + id UUID NOT NULL, + underlay_address INET NOT NULL, + zone_type omicron.public.zone_type NOT NULL, + + primary_service_ip INET NOT NULL, + primary_service_port INT4 + CHECK (primary_service_port BETWEEN 0 AND 65535) + NOT NULL, + + second_service_ip INET, + second_service_port INT4 + CHECK (second_service_port IS NULL + OR second_service_port BETWEEN 0 AND 65535), + + dataset_zpool_name TEXT, + + nic_id UUID, + + dns_gz_address INET, + dns_gz_address_index INT8, + + ntp_ntp_servers TEXT[], + ntp_dns_servers INET[], + ntp_domain TEXT, + + nexus_external_tls BOOLEAN, + nexus_external_dns_servers INET ARRAY, + + snat_ip INET, + snat_first_port INT4 + CHECK (snat_first_port IS NULL OR snat_first_port BETWEEN 0 AND 65535), + snat_last_port INT4 + CHECK (snat_last_port IS NULL OR snat_last_port BETWEEN 0 AND 65535), + + PRIMARY KEY (inv_collection_id, id) +); diff --git a/schema/crdb/22.0.0/up06.sql b/schema/crdb/22.0.0/up06.sql new file mode 100644 index 0000000000..3d50bcfefd --- /dev/null +++ b/schema/crdb/22.0.0/up06.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.inv_omicron_zone_nic ( + inv_collection_id UUID NOT NULL, + id UUID NOT NULL, + name TEXT NOT NULL, + ip INET NOT NULL, + mac INT8 NOT NULL, + subnet INET NOT NULL, + vni INT8 NOT NULL, + is_primary BOOLEAN NOT NULL, + slot INT2 NOT NULL, + + PRIMARY KEY (inv_collection_id, id) +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index cc61148048..57ce791a03 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -2916,6 +2916,161 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_root_of_trust_page ( PRIMARY KEY (inv_collection_id, hw_baseboard_id, which) ); +CREATE TYPE IF NOT EXISTS omicron.public.sled_role AS ENUM ( + -- this sled is directly attached to a Sidecar + 'scrimlet', + -- everything else + 'gimlet' +); + +-- observations from and about sled agents +CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_agent ( + -- where this observation came from + -- (foreign key into `inv_collection` table) + inv_collection_id UUID NOT NULL, + -- when this observation was made + time_collected TIMESTAMPTZ NOT NULL, + -- URL of the sled agent that reported this data + source TEXT NOT NULL, + + -- unique id for this sled (should be foreign keys into `sled` table, though + -- it's conceivable a sled will report an id that we don't know about) + sled_id UUID NOT NULL, + + -- which system this sled agent reports it's running on + -- (foreign key into `hw_baseboard_id` table) + -- This is optional because dev/test systems support running on non-Oxide + -- hardware. + hw_baseboard_id UUID, + + -- Many of the following properties are duplicated from the `sled` table, + -- which predates the current inventory system. + sled_agent_ip INET NOT NULL, + sled_agent_port INT4 NOT NULL, + sled_role omicron.public.sled_role NOT NULL, + usable_hardware_threads INT8 + CHECK (usable_hardware_threads BETWEEN 0 AND 4294967295) NOT NULL, + usable_physical_ram INT8 NOT NULL, + reservoir_size INT8 CHECK (reservoir_size < usable_physical_ram) NOT NULL, + + PRIMARY KEY (inv_collection_id, sled_id) +); + +CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_omicron_zones ( + -- where this observation came from + -- (foreign key into `inv_collection` table) + inv_collection_id UUID NOT NULL, + -- when this observation was made + time_collected TIMESTAMPTZ NOT NULL, + -- URL of the sled agent that reported this data + source TEXT NOT NULL, + + -- unique id for this sled (should be foreign keys into `sled` table, though + -- it's conceivable a sled will report an id that we don't know about) + sled_id UUID NOT NULL, + + -- OmicronZonesConfig generation reporting these zones + generation INT8 NOT NULL, + + PRIMARY KEY (inv_collection_id, sled_id) +); + +CREATE TYPE IF NOT EXISTS omicron.public.zone_type AS ENUM ( + 'boundary_ntp', + 'clickhouse', + 'clickhouse_keeper', + 'cockroach_db', + 'crucible', + 'crucible_pantry', + 'external_dns', + 'internal_dns', + 'internal_ntp', + 'nexus', + 'oximeter' +); + +-- observations from sled agents about Omicron-managed zones +CREATE TABLE IF NOT EXISTS omicron.public.inv_omicron_zone ( + -- where this observation came from + -- (foreign key into `inv_collection` table) + inv_collection_id UUID NOT NULL, + + -- unique id for this sled (should be foreign keys into `sled` table, though + -- it's conceivable a sled will report an id that we don't know about) + sled_id UUID NOT NULL, + + -- unique id for this zone + id UUID NOT NULL, + underlay_address INET NOT NULL, + zone_type omicron.public.zone_type NOT NULL, + + -- SocketAddr of the "primary" service for this zone + -- (what this describes varies by zone type, but all zones have at least one + -- service in them) + primary_service_ip INET NOT NULL, + primary_service_port INT4 + CHECK (primary_service_port BETWEEN 0 AND 65535) + NOT NULL, + + -- The remaining properties may be NULL for different kinds of zones. The + -- specific constraints are not enforced at the database layer, basically + -- because it's really complicated to do that and it's not obvious that it's + -- worthwhile. + + -- Some zones have a second service. Like the primary one, the meaning of + -- this is zone-type-dependent. + second_service_ip INET, + second_service_port INT4 + CHECK (second_service_port IS NULL + OR second_service_port BETWEEN 0 AND 65535), + + -- Zones may have an associated dataset. They're currently always on a U.2. + -- The only thing we need to identify it here is the name of the zpool that + -- it's on. + dataset_zpool_name TEXT, + + -- Zones with external IPs have an associated NIC and sockaddr for listening + -- (first is a foreign key into `inv_omicron_zone_nic`) + nic_id UUID, + + -- Properties for internal DNS servers + -- address attached to this zone from outside the sled's subnet + dns_gz_address INET, + dns_gz_address_index INT8, + + -- Properties common to both kinds of NTP zones + ntp_ntp_servers TEXT[], + ntp_dns_servers INET[], + ntp_domain TEXT, + + -- Properties specific to Nexus zones + nexus_external_tls BOOLEAN, + nexus_external_dns_servers INET ARRAY, + + -- Source NAT configuration (currently used for boundary NTP only) + snat_ip INET, + snat_first_port INT4 + CHECK (snat_first_port IS NULL OR snat_first_port BETWEEN 0 AND 65535), + snat_last_port INT4 + CHECK (snat_last_port IS NULL OR snat_last_port BETWEEN 0 AND 65535), + + PRIMARY KEY (inv_collection_id, id) +); + +CREATE TABLE IF NOT EXISTS omicron.public.inv_omicron_zone_nic ( + inv_collection_id UUID NOT NULL, + id UUID NOT NULL, + name TEXT NOT NULL, + ip INET NOT NULL, + mac INT8 NOT NULL, + subnet INET NOT NULL, + vni INT8 NOT NULL, + is_primary BOOLEAN NOT NULL, + slot INT2 NOT NULL, + + PRIMARY KEY (inv_collection_id, id) +); + /*******************************************************************/ /* @@ -3096,7 +3251,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '21.0.0', NULL) + ( TRUE, NOW(), NOW(), '22.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/sled-agent/src/bin/sled-agent-sim.rs b/sled-agent/src/bin/sled-agent-sim.rs index ee0ebda71e..4b3bc9e432 100644 --- a/sled-agent/src/bin/sled-agent-sim.rs +++ b/sled-agent/src/bin/sled-agent-sim.rs @@ -12,15 +12,15 @@ use clap::Parser; use dropshot::ConfigDropshot; use dropshot::ConfigLogging; use dropshot::ConfigLoggingLevel; -use dropshot::HandlerTaskMode; use nexus_client::types as NexusTypes; use omicron_common::cmd::fatal; use omicron_common::cmd::CmdError; use omicron_sled_agent::sim::RssArgs; use omicron_sled_agent::sim::{ - run_standalone_server, Config, ConfigHardware, ConfigStorage, - ConfigUpdates, ConfigZpool, SimMode, + run_standalone_server, Config, ConfigHardware, ConfigStorage, ConfigZpool, + SimMode, }; +use sled_hardware::Baseboard; use std::net::SocketAddr; use std::net::SocketAddrV6; use uuid::Uuid; @@ -98,26 +98,31 @@ async fn do_run() -> Result<(), CmdError> { let tmp = camino_tempfile::tempdir() .map_err(|e| CmdError::Failure(anyhow!(e)))?; let config = Config { - id: args.uuid, - sim_mode: args.sim_mode, - nexus_address: args.nexus_addr, dropshot: ConfigDropshot { bind_address: args.sled_agent_addr.into(), - request_body_max_bytes: 1024 * 1024, - default_handler_task_mode: HandlerTaskMode::Detached, + ..Default::default() }, - log: ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info }, storage: ConfigStorage { // Create 10 "virtual" U.2s, with 1 TB of storage. zpools: vec![ConfigZpool { size: 1 << 40 }; 10], ip: (*args.sled_agent_addr.ip()).into(), }, - updates: ConfigUpdates { zone_artifact_path: tmp.path().to_path_buf() }, hardware: ConfigHardware { hardware_threads: 32, physical_ram: 64 * (1 << 30), reservoir_ram: 32 * (1 << 30), + baseboard: Baseboard::Gimlet { + identifier: format!("sim-{}", args.uuid), + model: String::from("sim-gimlet"), + revision: 3, + }, }, + ..Config::for_testing( + args.uuid, + args.sim_mode, + Some(args.nexus_addr), + Some(tmp.path()), + ) }; let tls_certificate = match (args.rss_tls_cert, args.rss_tls_key) { @@ -145,5 +150,9 @@ async fn do_run() -> Result<(), CmdError> { tls_certificate, }; - run_standalone_server(&config, &rss_args).await.map_err(CmdError::Failure) + let config_logging = + ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info }; + run_standalone_server(&config, &config_logging, &rss_args) + .await + .map_err(CmdError::Failure) } diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 26a0d2ddc2..39d1ae26a0 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -10,9 +10,9 @@ use crate::bootstrap::params::AddSledRequest; use crate::params::{ CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, - InstancePutStateResponse, InstanceUnregisterResponse, OmicronZonesConfig, - SledRole, TimeSync, VpcFirewallRulesEnsureBody, ZoneBundleId, - ZoneBundleMetadata, Zpool, + InstancePutStateResponse, InstanceUnregisterResponse, Inventory, + OmicronZonesConfig, SledRole, TimeSync, VpcFirewallRulesEnsureBody, + ZoneBundleId, ZoneBundleMetadata, Zpool, }; use crate::sled_agent::Error as SledAgentError; use crate::zone_bundle; @@ -82,6 +82,7 @@ pub fn api() -> SledApiDescription { api.register(host_os_write_start)?; api.register(host_os_write_status_get)?; api.register(host_os_write_status_delete)?; + api.register(inventory)?; Ok(()) } @@ -925,3 +926,15 @@ async fn host_os_write_status_delete( .map_err(|err| HttpError::from(&err))?; Ok(HttpResponseUpdatedNoContent()) } + +/// Fetch basic information about this sled +#[endpoint { + method = GET, + path = "/inventory", +}] +async fn inventory( + request_context: RequestContext, +) -> Result, HttpError> { + let sa = request_context.context(); + Ok(HttpResponseOk(sa.inventory()?)) +} diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index a7d91e2b93..41fc84504e 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -10,6 +10,7 @@ pub use illumos_utils::opte::params::DhcpConfig; pub use illumos_utils::opte::params::VpcFirewallRule; pub use illumos_utils::opte::params::VpcFirewallRulesEnsureBody; use illumos_utils::zpool::ZpoolName; +use omicron_common::api::external::ByteCount; use omicron_common::api::external::Generation; use omicron_common::api::internal::nexus::{ DiskRuntimeState, InstanceProperties, InstanceRuntimeState, @@ -20,6 +21,7 @@ use omicron_common::api::internal::shared::{ }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use sled_hardware::Baseboard; pub use sled_hardware::DendriteAsic; use sled_storage::dataset::DatasetKind; use sled_storage::dataset::DatasetName; @@ -805,16 +807,6 @@ pub struct TimeSync { pub correction: f64, } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] -#[serde(rename_all = "snake_case")] -pub enum SledRole { - /// The sled is a general compute sled. - Gimlet, - /// The sled is attached to the network switch, and has additional - /// responsibilities. - Scrimlet, -} - /// Parameters used to update the zone bundle cleanup context. #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] pub struct CleanupContextUpdate { @@ -825,3 +817,20 @@ pub struct CleanupContextUpdate { /// The new limit on the underlying dataset quota allowed for bundles. pub storage_limit: Option, } + +// Our SledRole and Baseboard types do not have to be identical to the Nexus +// ones, but they generally should be, and this avoids duplication. If it +// becomes easier to maintain a separate copy, we should do that. +pub type SledRole = nexus_client::types::SledRole; + +/// Identity and basic status information about this sled agent +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct Inventory { + pub sled_id: Uuid, + pub sled_agent_address: SocketAddrV6, + pub sled_role: SledRole, + pub baseboard: Baseboard, + pub usable_hardware_threads: u32, + pub usable_physical_ram: ByteCount, + pub reservoir_size: ByteCount, +} diff --git a/sled-agent/src/sim/config.rs b/sled-agent/src/sim/config.rs index 62012a7109..81e11dc1c2 100644 --- a/sled-agent/src/sim/config.rs +++ b/sled-agent/src/sim/config.rs @@ -5,13 +5,22 @@ //! Interfaces for working with sled agent configuration use crate::updates::ConfigUpdates; +use camino::Utf8Path; use dropshot::ConfigDropshot; -use dropshot::ConfigLogging; use serde::Deserialize; use serde::Serialize; +pub use sled_hardware::Baseboard; +use std::net::Ipv6Addr; use std::net::{IpAddr, SocketAddr}; use uuid::Uuid; +/// The reported amount of hardware threads for an emulated sled agent. +pub const TEST_HARDWARE_THREADS: u32 = 16; +/// The reported amount of physical RAM for an emulated sled agent. +pub const TEST_PHYSICAL_RAM: u64 = 32 * (1 << 30); +/// The reported amount of VMM reservoir RAM for an emulated sled agent. +pub const TEST_RESERVOIR_RAM: u64 = 16 * (1 << 30); + /// How a [`SledAgent`](`super::sled_agent::SledAgent`) simulates object states and /// transitions #[derive(Copy, Clone, Debug, Deserialize, PartialEq, Serialize)] @@ -47,6 +56,7 @@ pub struct ConfigHardware { pub hardware_threads: u32, pub physical_ram: u64, pub reservoir_ram: u64, + pub baseboard: Baseboard, } /// Configuration for a sled agent @@ -60,8 +70,6 @@ pub struct Config { pub nexus_address: SocketAddr, /// configuration for the sled agent dropshot server pub dropshot: ConfigDropshot, - /// configuration for the sled agent debug log - pub log: ConfigLogging, /// configuration for the sled agent's storage pub storage: ConfigStorage, /// configuration for the sled agent's updates @@ -69,3 +77,49 @@ pub struct Config { /// configuration to emulate the sled agent's hardware pub hardware: ConfigHardware, } + +impl Config { + pub fn for_testing( + id: Uuid, + sim_mode: SimMode, + nexus_address: Option, + update_directory: Option<&Utf8Path>, + ) -> Config { + // This IP range is guaranteed by RFC 6666 to discard traffic. + // For tests that don't use a Nexus, we use this address to simulate a + // non-functioning Nexus. + let nexus_address = + nexus_address.unwrap_or_else(|| "[100::1]:12345".parse().unwrap()); + // If the caller doesn't care to provide a directory in which to put + // updates, make up a path that doesn't exist. + let update_directory = + update_directory.unwrap_or_else(|| "/nonexistent".into()); + Config { + id, + sim_mode, + nexus_address, + dropshot: ConfigDropshot { + bind_address: SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 0), + request_body_max_bytes: 1024 * 1024, + ..Default::default() + }, + storage: ConfigStorage { + zpools: vec![], + ip: IpAddr::from(Ipv6Addr::LOCALHOST), + }, + updates: ConfigUpdates { + zone_artifact_path: update_directory.to_path_buf(), + }, + hardware: ConfigHardware { + hardware_threads: TEST_HARDWARE_THREADS, + physical_ram: TEST_PHYSICAL_RAM, + reservoir_ram: TEST_RESERVOIR_RAM, + baseboard: Baseboard::Gimlet { + identifier: format!("sim-{}", id), + model: String::from("sim-gimlet"), + revision: 3, + }, + }, + } + } +} diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index f77da11b0e..e5d7752511 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -10,7 +10,7 @@ use crate::bootstrap::early_networking::{ use crate::params::{ DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, InstancePutStateResponse, InstanceUnregisterResponse, - VpcFirewallRulesEnsureBody, + Inventory, OmicronZonesConfig, VpcFirewallRulesEnsureBody, }; use dropshot::endpoint; use dropshot::ApiDescription; @@ -56,6 +56,9 @@ pub fn api() -> SledApiDescription { api.register(uplink_ensure)?; api.register(read_network_bootstore_config)?; api.register(write_network_bootstore_config)?; + api.register(inventory)?; + api.register(omicron_zones_get)?; + api.register(omicron_zones_put)?; Ok(()) } @@ -384,3 +387,43 @@ async fn write_network_bootstore_config( ) -> Result { Ok(HttpResponseUpdatedNoContent()) } + +/// Fetch basic information about this sled +#[endpoint { + method = GET, + path = "/inventory", +}] +async fn inventory( + rqctx: RequestContext>, +) -> Result, HttpError> { + let sa = rqctx.context(); + Ok(HttpResponseOk( + sa.inventory(rqctx.server.local_addr) + .map_err(|e| HttpError::for_internal_error(format!("{:#}", e)))?, + )) +} + +#[endpoint { + method = GET, + path = "/omicron-zones", +}] +async fn omicron_zones_get( + rqctx: RequestContext>, +) -> Result, HttpError> { + let sa = rqctx.context(); + Ok(HttpResponseOk(sa.omicron_zones_list().await)) +} + +#[endpoint { + method = PUT, + path = "/omicron-zones", +}] +async fn omicron_zones_put( + rqctx: RequestContext>, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + let body_args = body.into_inner(); + sa.omicron_zones_ensure(body_args).await; + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/sim/mod.rs b/sled-agent/src/sim/mod.rs index 8a730d5988..14d980cf79 100644 --- a/sled-agent/src/sim/mod.rs +++ b/sled-agent/src/sim/mod.rs @@ -17,6 +17,9 @@ mod sled_agent; mod storage; pub use crate::updates::ConfigUpdates; -pub use config::{Config, ConfigHardware, ConfigStorage, ConfigZpool, SimMode}; +pub use config::{ + Baseboard, Config, ConfigHardware, ConfigStorage, ConfigZpool, SimMode, + TEST_HARDWARE_THREADS, TEST_RESERVOIR_RAM, +}; pub use server::{run_standalone_server, RssArgs, Server}; pub use sled_agent::SledAgent; diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 1f2fe8e1d8..b214667631 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -50,6 +50,7 @@ impl Server { pub async fn start( config: &Config, log: &Logger, + wait_for_nexus: bool, ) -> Result { info!(log, "setting up sled agent server"); @@ -87,49 +88,61 @@ impl Server { // TODO-robustness if this returns a 400 error, we probably want to // return a permanent error from the `notify_nexus` closure. let sa_address = http_server.local_addr(); - let notify_nexus = || async { - debug!(log, "contacting server nexus"); - nexus_client - .sled_agent_put( - &config.id, - &NexusTypes::SledAgentStartupInfo { - sa_address: sa_address.to_string(), - role: NexusTypes::SledRole::Scrimlet, - baseboard: NexusTypes::Baseboard { - serial_number: format!( - "sim-{}", - &config.id.to_string()[0..8] - ), - part_number: String::from("Unknown"), - revision: 0, + let config_clone = config.clone(); + let log_clone = log.clone(); + let task = tokio::spawn(async move { + let config = config_clone; + let log = log_clone; + let nexus_client = nexus_client.clone(); + let notify_nexus = || async { + debug!(log, "contacting server nexus"); + nexus_client + .sled_agent_put( + &config.id, + &NexusTypes::SledAgentStartupInfo { + sa_address: sa_address.to_string(), + role: NexusTypes::SledRole::Scrimlet, + baseboard: NexusTypes::Baseboard { + serial_number: format!( + "sim-{}", + &config.id.to_string()[0..8] + ), + part_number: String::from("Unknown"), + revision: 0, + }, + usable_hardware_threads: config + .hardware + .hardware_threads, + usable_physical_ram: + NexusTypes::ByteCount::try_from( + config.hardware.physical_ram, + ) + .unwrap(), + reservoir_size: NexusTypes::ByteCount::try_from( + config.hardware.reservoir_ram, + ) + .unwrap(), }, - usable_hardware_threads: config - .hardware - .hardware_threads, - usable_physical_ram: NexusTypes::ByteCount::try_from( - config.hardware.physical_ram, - ) - .unwrap(), - reservoir_size: NexusTypes::ByteCount::try_from( - config.hardware.reservoir_ram, - ) - .unwrap(), - }, - ) - .await - .map_err(BackoffError::transient) - }; - let log_notification_failure = |error, delay| { - warn!(log, "failed to contact nexus, will retry in {:?}", delay; - "error" => ?error); - }; - retry_notify( - retry_policy_internal_service_aggressive(), - notify_nexus, - log_notification_failure, - ) - .await - .expect("Expected an infinite retry loop contacting Nexus"); + ) + .await + .map_err(BackoffError::transient) + }; + let log_notification_failure = |error, delay| { + warn!(log, "failed to contact nexus, will retry in {:?}", delay; + "error" => ?error); + }; + retry_notify( + retry_policy_internal_service_aggressive(), + notify_nexus, + log_notification_failure, + ) + .await + .expect("Expected an infinite retry loop contacting Nexus"); + }); + + if wait_for_nexus { + task.await.unwrap(); + } let mut datasets = vec![]; // Create all the Zpools requested by the config, and allocate a single @@ -262,11 +275,11 @@ pub struct RssArgs { /// - Performs handoff to Nexus pub async fn run_standalone_server( config: &Config, + logging: &dropshot::ConfigLogging, rss_args: &RssArgs, ) -> Result<(), anyhow::Error> { let (drain, registration) = slog_dtrace::with_drain( - config - .log + logging .to_logger("sled-agent") .map_err(|message| anyhow!("initializing logger: {}", message))?, ); @@ -280,7 +293,7 @@ pub async fn run_standalone_server( } // Start the sled agent - let mut server = Server::start(config, &log).await?; + let mut server = Server::start(config, &log, true).await?; info!(log, "sled agent started successfully"); // Start the Internal DNS server diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index a16049dd2f..8a76bf6abc 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -10,41 +10,42 @@ use super::disk::SimDisk; use super::instance::SimInstance; use super::storage::CrucibleData; use super::storage::Storage; - use crate::nexus::NexusClient; use crate::params::{ DiskStateRequested, InstanceHardware, InstanceMigrationSourceParams, InstancePutStateResponse, InstanceStateRequested, - InstanceUnregisterResponse, + InstanceUnregisterResponse, Inventory, OmicronZonesConfig, SledRole, }; use crate::sim::simulatable::Simulatable; use crate::updates::UpdateManager; +use anyhow::bail; +use anyhow::Context; +use dropshot::HttpServer; use futures::lock::Mutex; -use omicron_common::api::external::{DiskState, Error, ResourceType}; +use illumos_utils::opte::params::{ + DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost, +}; +use nexus_client::types::PhysicalDiskKind; +use omicron_common::address::PROPOLIS_PORT; +use omicron_common::api::external::{ + ByteCount, DiskState, Error, Generation, ResourceType, +}; use omicron_common::api::internal::nexus::{ DiskRuntimeState, SledInstanceState, }; use omicron_common::api::internal::nexus::{ InstanceRuntimeState, VmmRuntimeState, }; -use slog::Logger; -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; -use std::sync::Arc; -use uuid::Uuid; - -use std::collections::HashMap; -use std::str::FromStr; - -use dropshot::HttpServer; -use illumos_utils::opte::params::{ - DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost, -}; -use nexus_client::types::PhysicalDiskKind; -use omicron_common::address::PROPOLIS_PORT; use propolis_client::{ types::VolumeConstructionRequest, Client as PropolisClient, }; use propolis_mock_server::Context as PropolisContext; +use slog::Logger; +use std::collections::HashMap; +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::str::FromStr; +use std::sync::Arc; +use uuid::Uuid; /// Simulates management of the control plane on a sled /// @@ -68,7 +69,8 @@ pub struct SledAgent { pub v2p_mappings: Mutex>>, mock_propolis: Mutex>, PropolisClient)>>, - + config: Config, + fake_zones: Mutex, instance_ensure_state_error: Mutex>, } @@ -161,6 +163,11 @@ impl SledAgent { disk_id_to_region_ids: Mutex::new(HashMap::new()), v2p_mappings: Mutex::new(HashMap::new()), mock_propolis: Mutex::new(None), + config: config.clone(), + fake_zones: Mutex::new(OmicronZonesConfig { + generation: Generation::new(), + zones: vec![], + }), instance_ensure_state_error: Mutex::new(None), }) } @@ -665,4 +672,39 @@ impl SledAgent { *mock_lock = Some((srv, client)); Ok(()) } + + pub fn inventory(&self, addr: SocketAddr) -> anyhow::Result { + let sled_agent_address = match addr { + SocketAddr::V4(_) => { + bail!("sled_agent_ip must be v6 for inventory") + } + SocketAddr::V6(v6) => v6, + }; + Ok(Inventory { + sled_id: self.id, + sled_agent_address, + sled_role: SledRole::Gimlet, + baseboard: self.config.hardware.baseboard.clone(), + usable_hardware_threads: self.config.hardware.hardware_threads, + usable_physical_ram: ByteCount::try_from( + self.config.hardware.physical_ram, + ) + .context("usable_physical_ram")?, + reservoir_size: ByteCount::try_from( + self.config.hardware.reservoir_ram, + ) + .context("reservoir_size")?, + }) + } + + pub async fn omicron_zones_list(&self) -> OmicronZonesConfig { + self.fake_zones.lock().await.clone() + } + + pub async fn omicron_zones_ensure( + &self, + requested_zones: OmicronZonesConfig, + ) { + *self.fake_zones.lock().await = requested_zones; + } } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 621d003268..5bc0f8d257 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -18,8 +18,8 @@ use crate::nexus::{ConvertInto, NexusClientWithResolver, NexusRequestQueue}; use crate::params::{ DiskStateRequested, InstanceHardware, InstanceMigrationSourceParams, InstancePutStateResponse, InstanceStateRequested, - InstanceUnregisterResponse, OmicronZonesConfig, SledRole, TimeSync, - VpcFirewallRule, ZoneBundleMetadata, Zpool, + InstanceUnregisterResponse, Inventory, OmicronZonesConfig, SledRole, + TimeSync, VpcFirewallRule, ZoneBundleMetadata, Zpool, }; use crate::services::{self, ServiceManager}; use crate::storage_monitor::UnderlayAccess; @@ -42,7 +42,7 @@ use illumos_utils::zone::ZONE_PREFIX; use omicron_common::address::{ get_sled_address, get_switch_zone_address, Ipv6Subnet, SLED_PREFIX, }; -use omicron_common::api::external::Vni; +use omicron_common::api::external::{ByteCount, ByteCountRangeError, Vni}; use omicron_common::api::internal::nexus::ProducerEndpoint; use omicron_common::api::internal::nexus::ProducerKind; use omicron_common::api::internal::nexus::{ @@ -214,6 +214,35 @@ impl From for dropshot::HttpError { } } +/// Error returned by `SledAgent::inventory()` +#[derive(thiserror::Error, Debug)] +pub enum InventoryError { + // This error should be impossible because ByteCount supports values from + // [0, i64::MAX] and we don't have anything with that many bytes in the + // system. + #[error(transparent)] + BadByteCount(#[from] ByteCountRangeError), +} + +impl From for omicron_common::api::external::Error { + fn from(inventory_error: InventoryError) -> Self { + match inventory_error { + e @ InventoryError::BadByteCount(..) => { + omicron_common::api::external::Error::internal_error(&format!( + "{:#}", + e + )) + } + } + } +} + +impl From for dropshot::HttpError { + fn from(error: InventoryError) -> Self { + Self::from(omicron_common::api::external::Error::from(error)) + } +} + /// Describes an executing Sled Agent object. /// /// Contains both a connection to the Nexus, as well as managed instances. @@ -1056,6 +1085,37 @@ impl SledAgent { pub(crate) fn boot_disk_os_writer(&self) -> &BootDiskOsWriter { &self.inner.boot_disk_os_writer } + + /// Return basic information about ourselves: identity and status + /// + /// This is basically a GET version of the information we push to Nexus on + /// startup. + pub(crate) fn inventory(&self) -> Result { + let sled_id = self.inner.id; + let sled_agent_address = self.inner.sled_address(); + let is_scrimlet = self.inner.hardware.is_scrimlet(); + let baseboard = self.inner.hardware.baseboard(); + let usable_hardware_threads = + self.inner.hardware.online_processor_count(); + let usable_physical_ram = + self.inner.hardware.usable_physical_ram_bytes(); + let reservoir_size = self.inner.instances.reservoir_size(); + let sled_role = if is_scrimlet { + crate::params::SledRole::Scrimlet + } else { + crate::params::SledRole::Gimlet + }; + + Ok(Inventory { + sled_id, + sled_agent_address, + sled_role, + baseboard, + usable_hardware_threads, + usable_physical_ram: ByteCount::try_from(usable_physical_ram)?, + reservoir_size, + }) + } } async fn register_metric_producer_with_nexus(