From 8238581760c0eaa4e096a97e7e999b6b5cdcb62a Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Mon, 20 Nov 2023 12:02:39 -0500 Subject: [PATCH] [nexus] List all uninitialized sleds (#4504) As part of adding a sled to an already initialized rack, we need a way for operators to be able to list sleds that are not part of a rack. This PR adds an external nexus endpoint for doing just that. Like the `sleds` endpoint, this endpoint is not tied to a rack. The way this works is by looking at the SPs in the latest inventory collection and finding all the Baseboards that are not in the `sled` table in CRDB. A follow up commit will allow an operator to add uninitialized sleds to a rack with a new external nexus endpoint. --- dev-tools/omdb/src/bin/omdb/db.rs | 18 +- nexus/db-model/src/rack.rs | 19 +- .../db-queries/src/db/datastore/inventory.rs | 624 +++++++++--------- nexus/src/app/rack.rs | 71 +- nexus/src/external_api/http_entrypoints.rs | 24 +- nexus/tests/integration_tests/endpoints.rs | 9 + nexus/tests/output/nexus_tags.txt | 1 + nexus/types/src/external_api/views.rs | 30 +- openapi/nexus.json | 54 ++ 9 files changed, 530 insertions(+), 320 deletions(-) diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index efcefdea43..d009c05f86 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -53,7 +53,6 @@ use nexus_db_model::Zpool; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use nexus_db_queries::db::datastore::DataStoreConnection; -use nexus_db_queries::db::datastore::DataStoreInventoryTest; use nexus_db_queries::db::datastore::InstanceAndActiveVmm; use nexus_db_queries::db::identity::Asset; use nexus_db_queries::db::lookup::LookupPath; @@ -383,8 +382,13 @@ impl DbArgs { .await } DbCommands::Inventory(inventory_args) => { - cmd_db_inventory(&datastore, self.fetch_limit, inventory_args) - .await + cmd_db_inventory( + &opctx, + &datastore, + self.fetch_limit, + inventory_args, + ) + .await } DbCommands::Services(ServicesArgs { command: ServicesCommands::ListInstances, @@ -1751,6 +1755,7 @@ fn format_record(record: &DnsRecord) -> impl Display { // Inventory async fn cmd_db_inventory( + opctx: &OpContext, datastore: &DataStore, limit: NonZeroU32, inventory_args: &InventoryArgs, @@ -1768,7 +1773,9 @@ async fn cmd_db_inventory( }) => cmd_db_inventory_collections_list(&conn, limit).await, InventoryCommands::Collections(CollectionsArgs { command: CollectionsCommands::Show(CollectionsShowArgs { id }), - }) => cmd_db_inventory_collections_show(datastore, id, limit).await, + }) => { + cmd_db_inventory_collections_show(opctx, datastore, id, limit).await + } } } @@ -1928,12 +1935,13 @@ async fn cmd_db_inventory_collections_list( } async fn cmd_db_inventory_collections_show( + opctx: &OpContext, datastore: &DataStore, id: Uuid, limit: NonZeroU32, ) -> Result<(), anyhow::Error> { let (collection, incomplete) = datastore - .inventory_collection_read_best_effort(id, limit) + .inventory_collection_read_best_effort(opctx, id, limit) .await .context("reading collection")?; if incomplete { diff --git a/nexus/db-model/src/rack.rs b/nexus/db-model/src/rack.rs index 580ec155b4..f2bc7528d2 100644 --- a/nexus/db-model/src/rack.rs +++ b/nexus/db-model/src/rack.rs @@ -4,8 +4,9 @@ use crate::schema::rack; use db_macros::Asset; -use ipnetwork::IpNetwork; +use ipnetwork::{IpNetwork, Ipv6Network}; use nexus_types::{external_api::views, identity::Asset}; +use omicron_common::api; use uuid::Uuid; /// Information about a local rack. @@ -28,6 +29,22 @@ impl Rack { rack_subnet: None, } } + + pub fn subnet(&self) -> Result { + match self.rack_subnet { + Some(IpNetwork::V6(subnet)) => Ok(subnet), + Some(IpNetwork::V4(_)) => { + return Err(api::external::Error::InternalError { + internal_message: "rack subnet not IPv6".into(), + }) + } + None => { + return Err(api::external::Error::InternalError { + internal_message: "rack subnet not set".into(), + }) + } + } + } } impl From for views::Rack { diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 114b9dbe31..b743d28ee8 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -10,8 +10,6 @@ use crate::db::error::public_error_from_diesel; use crate::db::error::ErrorHandler; use crate::db::queries::ALLOW_FULL_TABLE_SCAN_SQL; use crate::db::TransactionError; -use anyhow::anyhow; -use anyhow::bail; use anyhow::Context; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; @@ -44,6 +42,7 @@ use nexus_db_model::SwCaboose; use nexus_types::inventory::Collection; use omicron_common::api::external::Error; use omicron_common::api::external::InternalContext; +use omicron_common::bail_unless; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::num::NonZeroU32; @@ -798,54 +797,311 @@ impl DataStore { Ok(()) } -} -/// Extra interfaces that are not intended (and potentially unsafe) for use in -/// Nexus, but useful for testing and `omdb` -pub trait DataStoreInventoryTest: Send + Sync { - /// List all collections - /// - /// This does not paginate. - fn inventory_collections(&self) -> BoxFuture>>; + /// Attempt to read the latest collection while limiting queries to `limit` + /// records + pub async fn inventory_get_latest_collection( + &self, + opctx: &OpContext, + limit: NonZeroU32, + ) -> Result { + opctx.authorize(authz::Action::Read, &authz::INVENTORY).await?; + let conn = self.pool_connection_authorized(opctx).await?; + use db::schema::inv_collection::dsl; + let collection_id = dsl::inv_collection + .select(dsl::id) + .order_by(dsl::time_started.desc()) + .limit(1) + .first_async::(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; - /// Make a best effort to read the given collection while limiting queries - /// to `limit` results. Returns as much as it was able to get. The - /// returned bool indicates whether the returned collection might be - /// incomplete because the limit was reached. - fn inventory_collection_read_best_effort( + self.inventory_collection_read_all_or_nothing( + opctx, + collection_id, + limit, + ) + .await + } + + /// Attempt to read the given collection while limiting queries to `limit` + /// records and returning nothing if `limit` is not large enough. + async fn inventory_collection_read_all_or_nothing( &self, + opctx: &OpContext, id: Uuid, limit: NonZeroU32, - ) -> BoxFuture>; + ) -> Result { + let (collection, limit_reached) = self + .inventory_collection_read_best_effort(opctx, id, limit) + .await?; + bail_unless!( + !limit_reached, + "hit limit of {} records while loading collection", + limit + ); + Ok(collection) + } - /// Attempt to read the given collection while limiting queries to `limit` - /// records - fn inventory_collection_read_all_or_nothing( + /// Make a best effort to read the given collection while limiting queries + /// to `limit` results. Returns as much as it was able to get. The + /// returned bool indicates whether the returned collection might be + /// incomplete because the limit was reached. + pub async fn inventory_collection_read_best_effort( &self, + opctx: &OpContext, id: Uuid, limit: NonZeroU32, - ) -> BoxFuture> { - async move { - let (collection, limit_reached) = - self.inventory_collection_read_best_effort(id, limit).await?; - anyhow::ensure!( - !limit_reached, - "hit limit of {} records while loading collection", - limit + ) -> Result<(Collection, bool), Error> { + let conn = self.pool_connection_authorized(opctx).await?; + let sql_limit = i64::from(u32::from(limit)); + let usize_limit = usize::try_from(u32::from(limit)).unwrap(); + let mut limit_reached = false; + let (time_started, time_done, collector) = { + use db::schema::inv_collection::dsl; + + let collections = dsl::inv_collection + .filter(dsl::id.eq(id)) + .limit(2) + .select(InvCollection::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + bail_unless!(collections.len() == 1); + let collection = collections.into_iter().next().unwrap(); + ( + collection.time_started, + collection.time_done, + collection.collector, + ) + }; + + let errors: Vec = { + use db::schema::inv_collection_error::dsl; + dsl::inv_collection_error + .filter(dsl::inv_collection_id.eq(id)) + .order_by(dsl::idx) + .limit(sql_limit) + .select(InvCollectionError::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|e| e.message) + .collect() + }; + limit_reached = limit_reached || errors.len() == usize_limit; + + let sps: BTreeMap<_, _> = { + use db::schema::inv_service_processor::dsl; + dsl::inv_service_processor + .filter(dsl::inv_collection_id.eq(id)) + .limit(sql_limit) + .select(InvServiceProcessor::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|sp_row| { + let baseboard_id = sp_row.hw_baseboard_id; + ( + baseboard_id, + nexus_types::inventory::ServiceProcessor::from(sp_row), + ) + }) + .collect() + }; + limit_reached = limit_reached || sps.len() == usize_limit; + + let rots: BTreeMap<_, _> = { + use db::schema::inv_root_of_trust::dsl; + dsl::inv_root_of_trust + .filter(dsl::inv_collection_id.eq(id)) + .limit(sql_limit) + .select(InvRootOfTrust::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|rot_row| { + let baseboard_id = rot_row.hw_baseboard_id; + ( + baseboard_id, + nexus_types::inventory::RotState::from(rot_row), + ) + }) + .collect() + }; + limit_reached = limit_reached || rots.len() == usize_limit; + + // Collect the unique baseboard ids referenced by SPs and RoTs. + let baseboard_id_ids: BTreeSet<_> = + sps.keys().chain(rots.keys()).cloned().collect(); + // Fetch the corresponding baseboard records. + let baseboards_by_id: BTreeMap<_, _> = { + use db::schema::hw_baseboard_id::dsl; + dsl::hw_baseboard_id + .filter(dsl::id.eq_any(baseboard_id_ids)) + .limit(sql_limit) + .select(HwBaseboardId::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|bb| { + ( + bb.id, + Arc::new(nexus_types::inventory::BaseboardId::from(bb)), + ) + }) + .collect() + }; + limit_reached = limit_reached || baseboards_by_id.len() == usize_limit; + + // Having those, we can replace the keys in the maps above with + // references to the actual baseboard rather than the uuid. + let sps = sps + .into_iter() + .map(|(id, sp)| { + baseboards_by_id.get(&id).map(|bb| (bb.clone(), sp)).ok_or_else( + || { + Error::internal_error( + "missing baseboard that we should have fetched", + ) + }, + ) + }) + .collect::, _>>()?; + let rots = rots + .into_iter() + .map(|(id, rot)| { + baseboards_by_id + .get(&id) + .map(|bb| (bb.clone(), rot)) + .ok_or_else(|| { + Error::internal_error( + "missing baseboard that we should have fetched", + ) + }) + }) + .collect::, _>>()?; + + // Fetch records of cabooses found. + let inv_caboose_rows = { + use db::schema::inv_caboose::dsl; + dsl::inv_caboose + .filter(dsl::inv_collection_id.eq(id)) + .limit(sql_limit) + .select(InvCaboose::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })? + }; + limit_reached = limit_reached || inv_caboose_rows.len() == usize_limit; + + // Collect the unique sw_caboose_ids for those cabooses. + let sw_caboose_ids: BTreeSet<_> = inv_caboose_rows + .iter() + .map(|inv_caboose| inv_caboose.sw_caboose_id) + .collect(); + // Fetch the corresponing records. + let cabooses_by_id: BTreeMap<_, _> = { + use db::schema::sw_caboose::dsl; + dsl::sw_caboose + .filter(dsl::id.eq_any(sw_caboose_ids)) + .limit(sql_limit) + .select(SwCaboose::as_select()) + .load_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))? + .into_iter() + .map(|sw_caboose_row| { + ( + sw_caboose_row.id, + Arc::new(nexus_types::inventory::Caboose::from( + sw_caboose_row, + )), + ) + }) + .collect() + }; + limit_reached = limit_reached || cabooses_by_id.len() == usize_limit; + + // Assemble the lists of cabooses found. + let mut cabooses_found = BTreeMap::new(); + for c in inv_caboose_rows { + let by_baseboard = cabooses_found + .entry(nexus_types::inventory::CabooseWhich::from(c.which)) + .or_insert_with(BTreeMap::new); + let Some(bb) = baseboards_by_id.get(&c.hw_baseboard_id) else { + let msg = format!( + "unknown baseboard found in inv_caboose: {}", + c.hw_baseboard_id + ); + return Err(Error::internal_error(&msg)); + }; + let Some(sw_caboose) = cabooses_by_id.get(&c.sw_caboose_id) else { + let msg = format!( + "unknown caboose found in inv_caboose: {}", + c.sw_caboose_id + ); + return Err(Error::internal_error(&msg)); + }; + + let previous = by_baseboard.insert( + bb.clone(), + nexus_types::inventory::CabooseFound { + time_collected: c.time_collected, + source: c.source, + caboose: sw_caboose.clone(), + }, + ); + bail_unless!( + previous.is_none(), + "duplicate caboose found: {:?} baseboard {:?}", + c.which, + c.hw_baseboard_id ); - Ok(collection) } - .boxed() + + Ok(( + Collection { + id, + errors, + time_started, + time_done, + collector, + baseboards: baseboards_by_id.values().cloned().collect(), + cabooses: cabooses_by_id.values().cloned().collect(), + sps, + rots, + cabooses_found, + }, + limit_reached, + )) } } +/// Extra interfaces that are not intended (and potentially unsafe) for use in +/// Nexus, but useful for testing and `omdb` +pub trait DataStoreInventoryTest: Send + Sync { + /// List all collections + /// + /// This does not paginate. + fn inventory_collections(&self) -> BoxFuture>>; +} + impl DataStoreInventoryTest for DataStore { fn inventory_collections(&self) -> BoxFuture>> { async { let conn = self .pool_connection_for_tests() .await - .context("getting connectoin")?; + .context("getting connection")?; conn.transaction_async(|conn| async move { conn.batch_execute_async(ALLOW_FULL_TABLE_SCAN_SQL) .await @@ -863,257 +1119,11 @@ impl DataStoreInventoryTest for DataStore { } .boxed() } - - // This function could move into the datastore if it proves helpful. We'd - // need to work out how to report the usual type of Error. For now we don't - // need it so we limit its scope to the test suite. - fn inventory_collection_read_best_effort( - &self, - id: Uuid, - limit: NonZeroU32, - ) -> BoxFuture> { - async move { - let conn = &self - .pool_connection_for_tests() - .await - .context("getting connection")?; - let sql_limit = i64::from(u32::from(limit)); - let usize_limit = usize::try_from(u32::from(limit)).unwrap(); - let mut limit_reached = false; - let (time_started, time_done, collector) = { - use db::schema::inv_collection::dsl; - - let collections = dsl::inv_collection - .filter(dsl::id.eq(id)) - .limit(2) - .select(InvCollection::as_select()) - .load_async(&**conn) - .await - .context("loading collection")?; - anyhow::ensure!(collections.len() == 1); - let collection = collections.into_iter().next().unwrap(); - ( - collection.time_started, - collection.time_done, - collection.collector, - ) - }; - - let errors: Vec = { - use db::schema::inv_collection_error::dsl; - dsl::inv_collection_error - .filter(dsl::inv_collection_id.eq(id)) - .order_by(dsl::idx) - .limit(sql_limit) - .select(InvCollectionError::as_select()) - .load_async(&**conn) - .await - .context("loading collection errors")? - .into_iter() - .map(|e| e.message) - .collect() - }; - limit_reached = limit_reached || errors.len() == usize_limit; - - let sps: BTreeMap<_, _> = { - use db::schema::inv_service_processor::dsl; - dsl::inv_service_processor - .filter(dsl::inv_collection_id.eq(id)) - .limit(sql_limit) - .select(InvServiceProcessor::as_select()) - .load_async(&**conn) - .await - .context("loading service processors")? - .into_iter() - .map(|sp_row| { - let baseboard_id = sp_row.hw_baseboard_id; - ( - baseboard_id, - nexus_types::inventory::ServiceProcessor::from( - sp_row, - ), - ) - }) - .collect() - }; - limit_reached = limit_reached || sps.len() == usize_limit; - - let rots: BTreeMap<_, _> = { - use db::schema::inv_root_of_trust::dsl; - dsl::inv_root_of_trust - .filter(dsl::inv_collection_id.eq(id)) - .limit(sql_limit) - .select(InvRootOfTrust::as_select()) - .load_async(&**conn) - .await - .context("loading roots of trust")? - .into_iter() - .map(|rot_row| { - let baseboard_id = rot_row.hw_baseboard_id; - ( - baseboard_id, - nexus_types::inventory::RotState::from(rot_row), - ) - }) - .collect() - }; - limit_reached = limit_reached || rots.len() == usize_limit; - - // Collect the unique baseboard ids referenced by SPs and RoTs. - let baseboard_id_ids: BTreeSet<_> = - sps.keys().chain(rots.keys()).cloned().collect(); - // Fetch the corresponding baseboard records. - let baseboards_by_id: BTreeMap<_, _> = { - use db::schema::hw_baseboard_id::dsl; - dsl::hw_baseboard_id - .filter(dsl::id.eq_any(baseboard_id_ids)) - .limit(sql_limit) - .select(HwBaseboardId::as_select()) - .load_async(&**conn) - .await - .context("loading baseboards")? - .into_iter() - .map(|bb| { - ( - bb.id, - Arc::new( - nexus_types::inventory::BaseboardId::from(bb), - ), - ) - }) - .collect() - }; - limit_reached = - limit_reached || baseboards_by_id.len() == usize_limit; - - // Having those, we can replace the keys in the maps above with - // references to the actual baseboard rather than the uuid. - let sps = sps - .into_iter() - .map(|(id, sp)| { - baseboards_by_id - .get(&id) - .map(|bb| (bb.clone(), sp)) - .ok_or_else(|| { - anyhow!( - "missing baseboard that we should have fetched" - ) - }) - }) - .collect::, _>>()?; - let rots = - rots.into_iter() - .map(|(id, rot)| { - baseboards_by_id - .get(&id) - .map(|bb| (bb.clone(), rot)) - .ok_or_else(|| { - anyhow!("missing baseboard that we should have fetched") - }) - }) - .collect::, _>>()?; - - // Fetch records of cabooses found. - let inv_caboose_rows = { - use db::schema::inv_caboose::dsl; - dsl::inv_caboose - .filter(dsl::inv_collection_id.eq(id)) - .limit(sql_limit) - .select(InvCaboose::as_select()) - .load_async(&**conn) - .await - .context("loading inv_cabooses")? - }; - limit_reached = - limit_reached || inv_caboose_rows.len() == usize_limit; - - // Collect the unique sw_caboose_ids for those cabooses. - let sw_caboose_ids: BTreeSet<_> = inv_caboose_rows - .iter() - .map(|inv_caboose| inv_caboose.sw_caboose_id) - .collect(); - // Fetch the corresponing records. - let cabooses_by_id: BTreeMap<_, _> = { - use db::schema::sw_caboose::dsl; - dsl::sw_caboose - .filter(dsl::id.eq_any(sw_caboose_ids)) - .limit(sql_limit) - .select(SwCaboose::as_select()) - .load_async(&**conn) - .await - .context("loading sw_cabooses")? - .into_iter() - .map(|sw_caboose_row| { - ( - sw_caboose_row.id, - Arc::new(nexus_types::inventory::Caboose::from( - sw_caboose_row, - )), - ) - }) - .collect() - }; - limit_reached = - limit_reached || cabooses_by_id.len() == usize_limit; - - // Assemble the lists of cabooses found. - let mut cabooses_found = BTreeMap::new(); - for c in inv_caboose_rows { - let by_baseboard = cabooses_found - .entry(nexus_types::inventory::CabooseWhich::from(c.which)) - .or_insert_with(BTreeMap::new); - let Some(bb) = baseboards_by_id.get(&c.hw_baseboard_id) else { - bail!( - "unknown baseboard found in inv_caboose: {}", - c.hw_baseboard_id - ); - }; - let Some(sw_caboose) = cabooses_by_id.get(&c.sw_caboose_id) - else { - bail!( - "unknown caboose found in inv_caboose: {}", - c.sw_caboose_id - ); - }; - - let previous = by_baseboard.insert( - bb.clone(), - nexus_types::inventory::CabooseFound { - time_collected: c.time_collected, - source: c.source, - caboose: sw_caboose.clone(), - }, - ); - anyhow::ensure!( - previous.is_none(), - "duplicate caboose found: {:?} baseboard {:?}", - c.which, - c.hw_baseboard_id - ); - } - - Ok(( - Collection { - id, - errors, - time_started, - time_done, - collector, - baseboards: baseboards_by_id.values().cloned().collect(), - cabooses: cabooses_by_id.values().cloned().collect(), - sps, - rots, - cabooses_found, - }, - limit_reached, - )) - } - .boxed() - } } #[cfg(test)] mod test { + use crate::context::OpContext; use crate::db::datastore::datastore_test; use crate::db::datastore::inventory::DataStoreInventoryTest; use crate::db::datastore::DataStore; @@ -1136,11 +1146,14 @@ mod test { use uuid::Uuid; async fn read_collection( + opctx: &OpContext, datastore: &DataStore, id: Uuid, ) -> anyhow::Result { let limit = NonZeroU32::new(1000).unwrap(); - datastore.inventory_collection_read_all_or_nothing(id, limit).await + Ok(datastore + .inventory_collection_read_all_or_nothing(opctx, id, limit) + .await?) } async fn count_baseboards_cabooses( @@ -1186,9 +1199,10 @@ mod test { // Read it back. let conn = datastore.pool_connection_for_tests().await.unwrap(); - let collection_read = read_collection(&datastore, collection1.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection1.id) + .await + .expect("failed to read collection back"); assert_eq!(collection1, collection_read); // There ought to be no baseboards or cabooses in the databases from @@ -1208,9 +1222,10 @@ mod test { .inventory_insert_collection(&opctx, &collection2) .await .expect("failed to insert collection"); - let collection_read = read_collection(&datastore, collection2.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection2.id) + .await + .expect("failed to read collection back"); assert_eq!(collection2, collection_read); // Verify that we have exactly the set of cabooses and baseboards in the // databases that came from this first non-empty collection. @@ -1221,6 +1236,18 @@ mod test { assert_eq!(collection2.baseboards.len(), nbaseboards); assert_eq!(collection2.cabooses.len(), ncabooses); + // Check that we get an error on the limit being reached for + // `read_all_or_nothing` + let limit = NonZeroU32::new(1).unwrap(); + assert!(datastore + .inventory_collection_read_all_or_nothing( + &opctx, + collection2.id, + limit + ) + .await + .is_err()); + // Now insert an equivalent collection again. Verify the distinct // baseboards and cabooses again. This is important: the insertion // process should re-use the baseboards and cabooses from the previous @@ -1231,9 +1258,10 @@ mod test { .inventory_insert_collection(&opctx, &collection3) .await .expect("failed to insert collection"); - let collection_read = read_collection(&datastore, collection3.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection3.id) + .await + .expect("failed to read collection back"); assert_eq!(collection3, collection_read); // Verify that we have the same number of cabooses and baseboards, since // those didn't change. @@ -1275,9 +1303,10 @@ mod test { .inventory_insert_collection(&opctx, &collection4) .await .expect("failed to insert collection"); - let collection_read = read_collection(&datastore, collection4.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection4.id) + .await + .expect("failed to read collection back"); assert_eq!(collection4, collection_read); // Verify the number of baseboards and collections again. assert_eq!( @@ -1302,9 +1331,10 @@ mod test { .inventory_insert_collection(&opctx, &collection5) .await .expect("failed to insert collection"); - let collection_read = read_collection(&datastore, collection5.id) - .await - .expect("failed to read collection back"); + let collection_read = + read_collection(&opctx, &datastore, collection5.id) + .await + .expect("failed to read collection back"); assert_eq!(collection5, collection_read); assert_eq!(collection5.baseboards.len(), collection3.baseboards.len()); assert_eq!(collection5.cabooses.len(), collection3.cabooses.len()); @@ -1433,19 +1463,19 @@ mod test { ); // If we try to fetch a pruned collection, we should get nothing. - let _ = read_collection(&datastore, collection4.id) + let _ = read_collection(&opctx, &datastore, collection4.id) .await .expect_err("unexpectedly read pruned collection"); // But we should still be able to fetch the collections that do exist. let collection_read = - read_collection(&datastore, collection5.id).await.unwrap(); + read_collection(&opctx, &datastore, collection5.id).await.unwrap(); assert_eq!(collection5, collection_read); let collection_read = - read_collection(&datastore, collection6.id).await.unwrap(); + read_collection(&opctx, &datastore, collection6.id).await.unwrap(); assert_eq!(collection6, collection_read); let collection_read = - read_collection(&datastore, collection7.id).await.unwrap(); + read_collection(&opctx, &datastore, collection7.id).await.unwrap(); assert_eq!(collection7, collection_read); // We should prune more than one collection, if needed. We'll wind up diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 163f3bd5bb..1c2e49e260 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -9,6 +9,7 @@ use crate::external_api::params; use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; +use gateway_client::types::SpType; use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; @@ -31,6 +32,9 @@ use nexus_types::external_api::params::{ use nexus_types::external_api::shared::FleetRole; use nexus_types::external_api::shared::SiloIdentityMode; use nexus_types::external_api::shared::SiloRole; +use nexus_types::external_api::views; +use nexus_types::external_api::views::Baseboard; +use nexus_types::external_api::views::UninitializedSled; use nexus_types::internal_api::params::DnsRecord; use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::DataPageParams; @@ -51,6 +55,7 @@ use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; use std::net::Ipv4Addr; +use std::num::NonZeroU32; use std::str::FromStr; use uuid::Uuid; @@ -614,20 +619,7 @@ impl super::Nexus { opctx: &OpContext, ) -> Result { let rack = self.rack_lookup(opctx, &self.rack_id).await?; - - let subnet = match rack.rack_subnet { - Some(IpNetwork::V6(subnet)) => subnet, - Some(IpNetwork::V4(_)) => { - return Err(Error::InternalError { - internal_message: "rack subnet not IPv6".into(), - }) - } - None => { - return Err(Error::InternalError { - internal_message: "rack subnet not set".into(), - }) - } - }; + let subnet = rack.subnet()?; let db_ports = self.active_port_settings(opctx).await?; let mut ports = Vec::new(); @@ -724,4 +716,55 @@ impl super::Nexus { Ok(result) } + + /// Return the list of sleds that are inserted into an initialized rack + /// but not yet initialized as part of a rack. + // + // TODO-multirack: We currently limit sleds to a single rack and we also + // retrieve the `rack_uuid` from the Nexus instance used. + pub(crate) async fn uninitialized_sled_list( + &self, + opctx: &OpContext, + ) -> ListResultVec { + // Grab the SPs from the last collection + let limit = NonZeroU32::new(50).unwrap(); + let collection = self + .db_datastore + .inventory_get_latest_collection(opctx, limit) + .await?; + let pagparams = DataPageParams { + marker: None, + direction: dropshot::PaginationOrder::Descending, + // TODO: This limit is only suitable for a single sled cluster + limit: NonZeroU32::new(32).unwrap(), + }; + let sleds = self.db_datastore.sled_list(opctx, &pagparams).await?; + + let mut uninitialized_sleds: Vec = collection + .sps + .into_iter() + .filter_map(|(k, v)| { + if v.sp_type == SpType::Sled { + Some(UninitializedSled { + baseboard: Baseboard { + serial: k.serial_number.clone(), + part: k.part_number.clone(), + revision: v.baseboard_revision.into(), + }, + rack_id: self.rack_id, + cubby: v.sp_slot, + }) + } else { + None + } + }) + .collect(); + + let sled_baseboards: BTreeSet = + sleds.into_iter().map(|s| views::Sled::from(s).baseboard).collect(); + + // Retain all sleds that exist but are not in the sled table + uninitialized_sleds.retain(|s| !sled_baseboards.contains(&s.baseboard)); + Ok(uninitialized_sleds) + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index eba97a88ec..428632bcf5 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -8,8 +8,8 @@ use super::{ console_api, device_auth, params, views::{ self, Certificate, Group, IdentityProvider, Image, IpPool, IpPoolRange, - PhysicalDisk, Project, Rack, Role, Silo, Sled, Snapshot, SshKey, User, - UserBuiltin, Vpc, VpcRouter, VpcSubnet, + PhysicalDisk, Project, Rack, Role, Silo, Sled, Snapshot, SshKey, + UninitializedSled, User, UserBuiltin, Vpc, VpcRouter, VpcSubnet, }, }; use crate::external_api::shared; @@ -222,6 +222,7 @@ pub(crate) fn external_api() -> NexusApiDescription { api.register(physical_disk_list)?; api.register(switch_list)?; api.register(switch_view)?; + api.register(uninitialized_sled_list)?; api.register(user_builtin_list)?; api.register(user_builtin_view)?; @@ -4382,6 +4383,25 @@ async fn rack_view( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } +/// List uninitialized sleds in a given rack +#[endpoint { + method = GET, + path = "/v1/system/hardware/uninitialized-sleds", + tags = ["system/hardware"] +}] +async fn uninitialized_sled_list( + rqctx: RequestContext>, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let sleds = nexus.uninitialized_sled_list(&opctx).await?; + Ok(HttpResponseOk(sleds)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + // Sleds /// List sleds diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 8fba22fb2f..64790c49c2 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -43,6 +43,8 @@ use std::str::FromStr; lazy_static! { pub static ref HARDWARE_RACK_URL: String = format!("/v1/system/hardware/racks/{}", RACK_UUID); + pub static ref HARDWARE_UNINITIALIZED_SLEDS: String = + format!("/v1/system/hardware/uninitialized-sleds"); pub static ref HARDWARE_SLED_URL: String = format!("/v1/system/hardware/sleds/{}", SLED_AGENT_UUID); pub static ref HARDWARE_SWITCH_URL: String = @@ -1564,6 +1566,13 @@ lazy_static! { allowed_methods: vec![AllowedMethod::Get], }, + VerifyEndpoint { + url: &HARDWARE_UNINITIALIZED_SLEDS, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![AllowedMethod::Get], + }, + VerifyEndpoint { url: "/v1/system/hardware/sleds", visibility: Visibility::Public, diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 8c5fe953e3..7f0c30c471 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -122,6 +122,7 @@ sled_physical_disk_list GET /v1/system/hardware/sleds/{sle sled_view GET /v1/system/hardware/sleds/{sled_id} switch_list GET /v1/system/hardware/switches switch_view GET /v1/system/hardware/switches/{switch_id} +uninitialized_sled_list GET /v1/system/hardware/uninitialized-sleds API operations found with tag "system/metrics" OPERATION ID METHOD URL PATH diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index ef3835c618..b34fc7a542 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -274,10 +274,38 @@ pub struct Rack { pub identity: AssetIdentityMetadata, } +/// View of a sled that has not been added to an initialized rack yet +#[derive( + Clone, + Debug, + Serialize, + Deserialize, + JsonSchema, + PartialOrd, + Ord, + PartialEq, + Eq, +)] +pub struct UninitializedSled { + pub baseboard: Baseboard, + pub rack_id: Uuid, + pub cubby: u16, +} + // FRUs /// Properties that uniquely identify an Oxide hardware component -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +#[derive( + Clone, + Debug, + Serialize, + Deserialize, + JsonSchema, + PartialOrd, + Ord, + PartialEq, + Eq, +)] pub struct Baseboard { pub serial: String, pub part: String, diff --git a/openapi/nexus.json b/openapi/nexus.json index 74162a9b2b..0d19e81d9a 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -4064,6 +4064,37 @@ } } }, + "/v1/system/hardware/uninitialized-sleds": { + "get": { + "tags": [ + "system/hardware" + ], + "summary": "List uninitialized sleds in a given rack", + "operationId": "uninitialized_sled_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_UninitializedSled", + "type": "array", + "items": { + "$ref": "#/components/schemas/UninitializedSled" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/system/identity-providers": { "get": { "tags": [ @@ -13939,6 +13970,29 @@ "vlan_id" ] }, + "UninitializedSled": { + "description": "View of a sled that has not been added to an initialized rack yet", + "type": "object", + "properties": { + "baseboard": { + "$ref": "#/components/schemas/Baseboard" + }, + "cubby": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "rack_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "baseboard", + "cubby", + "rack_id" + ] + }, "User": { "description": "View of a User", "type": "object",