diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 6b65eb87ec..ac5bad26f8 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -72,6 +72,7 @@ mod sled; mod sled_instance; mod sled_resource; mod sled_resource_kind; +mod sled_underlay_subnet_allocation; mod snapshot; mod ssh_key; mod switch; @@ -153,6 +154,7 @@ pub use sled::*; pub use sled_instance::*; pub use sled_resource::*; pub use sled_resource_kind::*; +pub use sled_underlay_subnet_allocation::*; pub use snapshot::*; pub use ssh_key::*; pub use switch::*; diff --git a/nexus/db-model/src/rack.rs b/nexus/db-model/src/rack.rs index f2bc7528d2..580ec155b4 100644 --- a/nexus/db-model/src/rack.rs +++ b/nexus/db-model/src/rack.rs @@ -4,9 +4,8 @@ use crate::schema::rack; use db_macros::Asset; -use ipnetwork::{IpNetwork, Ipv6Network}; +use ipnetwork::IpNetwork; use nexus_types::{external_api::views, identity::Asset}; -use omicron_common::api; use uuid::Uuid; /// Information about a local rack. @@ -29,22 +28,6 @@ impl Rack { rack_subnet: None, } } - - pub fn subnet(&self) -> Result { - match self.rack_subnet { - Some(IpNetwork::V6(subnet)) => Ok(subnet), - Some(IpNetwork::V4(_)) => { - return Err(api::external::Error::InternalError { - internal_message: "rack subnet not IPv6".into(), - }) - } - None => { - return Err(api::external::Error::InternalError { - internal_message: "rack subnet not set".into(), - }) - } - } - } } impl From for views::Rack { diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 7f7dd57027..afeac5e6cd 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -755,6 +755,16 @@ table! { } } +table! { + sled_underlay_subnet_allocation (rack_id, sled_id) { + rack_id -> Uuid, + sled_id -> Uuid, + subnet_octet -> Int2, + hw_baseboard_id -> Uuid, + } +} +allow_tables_to_appear_in_same_query!(rack, sled_underlay_subnet_allocation); + table! { switch (id) { id -> Uuid, @@ -1289,7 +1299,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(13, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(14, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/nexus/db-model/src/sled.rs b/nexus/db-model/src/sled.rs index ba572901c6..4c82aa5d23 100644 --- a/nexus/db-model/src/sled.rs +++ b/nexus/db-model/src/sled.rs @@ -8,7 +8,7 @@ use crate::ipv6; use crate::schema::{physical_disk, service, sled, zpool}; use chrono::{DateTime, Utc}; use db_macros::Asset; -use nexus_types::{external_api::views, identity::Asset}; +use nexus_types::{external_api::shared, external_api::views, identity::Asset}; use std::net::Ipv6Addr; use std::net::SocketAddrV6; use uuid::Uuid; @@ -88,7 +88,7 @@ impl From for views::Sled { Self { identity: sled.identity(), rack_id: sled.rack_id, - baseboard: views::Baseboard { + baseboard: shared::Baseboard { serial: sled.serial_number, part: sled.part_number, revision: sled.revision, diff --git a/nexus/db-model/src/sled_underlay_subnet_allocation.rs b/nexus/db-model/src/sled_underlay_subnet_allocation.rs new file mode 100644 index 0000000000..4da0bea669 --- /dev/null +++ b/nexus/db-model/src/sled_underlay_subnet_allocation.rs @@ -0,0 +1,16 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::schema::sled_underlay_subnet_allocation; +use uuid::Uuid; + +/// Underlay allocation for a sled added to an initialized rack +#[derive(Queryable, Insertable, Debug, Clone, Selectable)] +#[diesel(table_name = sled_underlay_subnet_allocation)] +pub struct SledUnderlaySubnetAllocation { + pub rack_id: Uuid, + pub sled_id: Uuid, + pub subnet_octet: i16, + pub hw_baseboard_id: Uuid, +} diff --git a/nexus/db-model/src/switch.rs b/nexus/db-model/src/switch.rs index c9db100b0a..159888d91e 100644 --- a/nexus/db-model/src/switch.rs +++ b/nexus/db-model/src/switch.rs @@ -2,7 +2,7 @@ use super::Generation; use crate::schema::switch; use chrono::{DateTime, Utc}; use db_macros::Asset; -use nexus_types::{external_api::views, identity::Asset}; +use nexus_types::{external_api::shared, external_api::views, identity::Asset}; use uuid::Uuid; /// Baseboard information about a switch. @@ -57,7 +57,7 @@ impl From for views::Switch { Self { identity: switch.identity(), rack_id: switch.rack_id, - baseboard: views::Baseboard { + baseboard: shared::Baseboard { serial: switch.serial_number, part: switch.part_number, revision: switch.revision, diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 28a438629e..31b24a7e75 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -7,6 +7,7 @@ use crate::authz; use crate::context::OpContext; use crate::db; use crate::db::error::public_error_from_diesel; +use crate::db::error::public_error_from_diesel_lookup; use crate::db::error::ErrorHandler; use crate::db::queries::ALLOW_FULL_TABLE_SCAN_SQL; use crate::db::TransactionError; @@ -21,6 +22,7 @@ use diesel::ExpressionMethods; use diesel::IntoSql; use diesel::JoinOnDsl; use diesel::NullableExpressionMethods; +use diesel::OptionalExtension; use diesel::QueryDsl; use diesel::Table; use futures::future::BoxFuture; @@ -42,9 +44,12 @@ use nexus_db_model::SpType; use nexus_db_model::SpTypeEnum; use nexus_db_model::SwCaboose; use nexus_db_model::SwRotPage; +use nexus_types::inventory::BaseboardId; use nexus_types::inventory::Collection; use omicron_common::api::external::Error; use omicron_common::api::external::InternalContext; +use omicron_common::api::external::LookupType; +use omicron_common::api::external::ResourceType; use omicron_common::bail_unless; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -910,30 +915,62 @@ impl DataStore { Ok(()) } + // Find the primary key for `hw_baseboard_id` given a `BaseboardId` + pub async fn find_hw_baseboard_id( + &self, + opctx: &OpContext, + baseboard_id: BaseboardId, + ) -> Result { + opctx.authorize(authz::Action::Read, &authz::INVENTORY).await?; + let conn = self.pool_connection_authorized(opctx).await?; + use db::schema::hw_baseboard_id::dsl; + dsl::hw_baseboard_id + .filter(dsl::serial_number.eq(baseboard_id.serial_number.clone())) + .filter(dsl::part_number.eq(baseboard_id.part_number.clone())) + .select(dsl::id) + .first_async::(&*conn) + .await + .map_err(|e| { + public_error_from_diesel_lookup( + e, + ResourceType::Sled, + &LookupType::ByCompositeId(format!("{baseboard_id:?}")), + ) + }) + } + /// Attempt to read the latest collection while limiting queries to `limit` /// records + /// + /// If there aren't any collections, return `Ok(None)`. pub async fn inventory_get_latest_collection( &self, opctx: &OpContext, limit: NonZeroU32, - ) -> Result { + ) -> Result, Error> { opctx.authorize(authz::Action::Read, &authz::INVENTORY).await?; let conn = self.pool_connection_authorized(opctx).await?; use db::schema::inv_collection::dsl; let collection_id = dsl::inv_collection .select(dsl::id) .order_by(dsl::time_started.desc()) - .limit(1) .first_async::(&*conn) .await + .optional() .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; - self.inventory_collection_read_all_or_nothing( - opctx, - collection_id, - limit, - ) - .await + let Some(collection_id) = collection_id else { + return Ok(None); + }; + + Ok(Some( + self.inventory_collection_read_all_or_nothing( + opctx, + collection_id, + limit, + ) + .await?, + )) } /// Attempt to read the given collection while limiting queries to `limit` @@ -1335,9 +1372,11 @@ mod test { use nexus_inventory::examples::Representative; use nexus_test_utils::db::test_setup_database; use nexus_test_utils::db::ALLOW_FULL_TABLE_SCAN_SQL; + use nexus_types::inventory::BaseboardId; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; use nexus_types::inventory::RotPageWhich; + use omicron_common::api::external::Error; use omicron_test_utils::dev; use std::num::NonZeroU32; use uuid::Uuid; @@ -1393,6 +1432,24 @@ mod test { } } + #[tokio::test] + async fn test_find_hw_baseboard_id_missing_returns_not_found() { + let logctx = dev::test_setup_log("inventory_insert"); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + let baseboard_id = BaseboardId { + serial_number: "some-serial".into(), + part_number: "some-part".into(), + }; + let err = datastore + .find_hw_baseboard_id(&opctx, baseboard_id) + .await + .unwrap_err(); + assert!(matches!(err, Error::ObjectNotFound { .. })); + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + /// Tests inserting several collections, reading them back, and making sure /// they look the same. #[tokio::test] diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 2cc5880470..e11377f11a 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -41,6 +41,7 @@ use nexus_db_model::InitialDnsGroup; use nexus_db_model::PasswordHashString; use nexus_db_model::SiloUser; use nexus_db_model::SiloUserPasswordHash; +use nexus_db_model::SledUnderlaySubnetAllocation; use nexus_types::external_api::params as external_params; use nexus_types::external_api::shared; use nexus_types::external_api::shared::IdentityType; @@ -55,6 +56,7 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_common::api::external::UpdateResult; +use omicron_common::bail_unless; use std::net::IpAddr; use uuid::Uuid; @@ -214,6 +216,126 @@ impl DataStore { Ok(()) } + // Return the subnet for the rack + pub async fn rack_subnet( + &self, + opctx: &OpContext, + rack_id: Uuid, + ) -> Result { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + let conn = self.pool_connection_authorized(opctx).await?; + use db::schema::rack::dsl; + // It's safe to unwrap the returned `rack_subnet` because + // we filter on `rack_subnet.is_not_null()` + let subnet = dsl::rack + .filter(dsl::id.eq(rack_id)) + .filter(dsl::rack_subnet.is_not_null()) + .select(dsl::rack_subnet) + .first_async::>(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + match subnet { + Some(subnet) => Ok(subnet), + None => Err(Error::internal_error( + "DB Error(bug): returned a null subnet for {rack_id}", + )), + } + } + + /// Allocate a rack subnet octet to a given sled + /// + /// 1. Find the existing allocations + /// 2. Calculate the new allocation + /// 3. Save the new allocation, if there isn't one for the given + /// `hw_baseboard_id` + /// 4. Return the new allocation + /// + // TODO: This could all actually be done in SQL using a `next_item` query. + // See https://github.com/oxidecomputer/omicron/issues/4544 + pub async fn allocate_sled_underlay_subnet_octets( + &self, + opctx: &OpContext, + rack_id: Uuid, + hw_baseboard_id: Uuid, + ) -> Result { + // Fetch all the existing allocations via self.rack_id + let allocations = self.rack_subnet_allocations(opctx, rack_id).await?; + + // Calculate the allocation for the new sled by choosing the minimum + // octet. The returned allocations are ordered by octet, so we will know + // when we have a free one. However, if we already have an allocation + // for the given sled then reuse that one. + const MIN_SUBNET_OCTET: i16 = 33; + let mut new_allocation = SledUnderlaySubnetAllocation { + rack_id, + sled_id: Uuid::new_v4(), + subnet_octet: MIN_SUBNET_OCTET, + hw_baseboard_id, + }; + let mut allocation_already_exists = false; + for allocation in allocations { + if allocation.hw_baseboard_id == new_allocation.hw_baseboard_id { + // We already have an allocation for this sled. + new_allocation = allocation; + allocation_already_exists = true; + break; + } + if allocation.subnet_octet == new_allocation.subnet_octet { + bail_unless!( + new_allocation.subnet_octet < 255, + "Too many sled subnets allocated" + ); + new_allocation.subnet_octet += 1; + } + } + + // Write the new allocation row to CRDB. The UNIQUE constraint + // on `subnet_octet` will prevent dueling administrators reusing + // allocations when sleds are being added. We will need another + // mechanism ala generation numbers when we must interleave additions + // and removals of sleds. + if !allocation_already_exists { + self.sled_subnet_allocation_insert(opctx, &new_allocation).await?; + } + + Ok(new_allocation) + } + + /// Return all current underlay allocations for the rack. + /// + /// Order allocations by `subnet_octet` + pub async fn rack_subnet_allocations( + &self, + opctx: &OpContext, + rack_id: Uuid, + ) -> Result, Error> { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + use db::schema::sled_underlay_subnet_allocation::dsl as subnet_dsl; + subnet_dsl::sled_underlay_subnet_allocation + .filter(subnet_dsl::rack_id.eq(rack_id)) + .select(SledUnderlaySubnetAllocation::as_select()) + .order_by(subnet_dsl::subnet_octet.asc()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + /// Store a new sled subnet allocation in the database + pub async fn sled_subnet_allocation_insert( + &self, + opctx: &OpContext, + allocation: &SledUnderlaySubnetAllocation, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + use db::schema::sled_underlay_subnet_allocation::dsl; + diesel::insert_into(dsl::sled_underlay_subnet_allocation) + .values(allocation.clone()) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) + } + // The following methods which return a `TxnError` take a `conn` parameter // which comes from the transaction created in `rack_set_initialized`. @@ -1518,4 +1640,136 @@ mod test { db.cleanup().await.unwrap(); logctx.cleanup_successful(); } + + #[tokio::test] + async fn rack_sled_subnet_allocations() { + let logctx = dev::test_setup_log("rack_sled_subnet_allocations"); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + let rack_id = Uuid::new_v4(); + + // Ensure we get an empty list when there are no allocations + let allocations = + datastore.rack_subnet_allocations(&opctx, rack_id).await.unwrap(); + assert!(allocations.is_empty()); + + // Add 5 allocations + for i in 0..5i16 { + let allocation = SledUnderlaySubnetAllocation { + rack_id, + sled_id: Uuid::new_v4(), + subnet_octet: 33 + i, + hw_baseboard_id: Uuid::new_v4(), + }; + datastore + .sled_subnet_allocation_insert(&opctx, &allocation) + .await + .unwrap(); + } + + // List all 5 allocations + let allocations = + datastore.rack_subnet_allocations(&opctx, rack_id).await.unwrap(); + + assert_eq!(5, allocations.len()); + + // Try to add another allocation for the same octet, but with a distinct + // sled_id. Ensure we get an error due to a unique constraint. + let mut should_fail_allocation = SledUnderlaySubnetAllocation { + rack_id, + sled_id: Uuid::new_v4(), + subnet_octet: 37, + hw_baseboard_id: Uuid::new_v4(), + }; + let _err = datastore + .sled_subnet_allocation_insert(&opctx, &should_fail_allocation) + .await + .unwrap_err(); + + // Adding an allocation for the same {rack_id, sled_id} pair fails + // the second time, even with a distinct subnet_epoch + let mut allocation = should_fail_allocation.clone(); + allocation.subnet_octet = 38; + datastore + .sled_subnet_allocation_insert(&opctx, &allocation) + .await + .unwrap(); + + should_fail_allocation.subnet_octet = 39; + should_fail_allocation.hw_baseboard_id = Uuid::new_v4(); + let _err = datastore + .sled_subnet_allocation_insert(&opctx, &should_fail_allocation) + .await + .unwrap_err(); + + // Allocations outside our expected range fail + let mut should_fail_allocation = SledUnderlaySubnetAllocation { + rack_id, + sled_id: Uuid::new_v4(), + subnet_octet: 32, + hw_baseboard_id: Uuid::new_v4(), + }; + let _err = datastore + .sled_subnet_allocation_insert(&opctx, &should_fail_allocation) + .await + .unwrap_err(); + should_fail_allocation.subnet_octet = 256; + let _err = datastore + .sled_subnet_allocation_insert(&opctx, &should_fail_allocation) + .await + .unwrap_err(); + + // We should have 6 allocations + let allocations = + datastore.rack_subnet_allocations(&opctx, rack_id).await.unwrap(); + + assert_eq!(6, allocations.len()); + assert_eq!( + vec![33, 34, 35, 36, 37, 38], + allocations.iter().map(|a| a.subnet_octet).collect::>() + ); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn allocate_sled_underlay_subnet_octets() { + let logctx = dev::test_setup_log("rack_sled_subnet_allocations"); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + let rack_id = Uuid::new_v4(); + + let mut allocated_octets = vec![]; + for _ in 0..5 { + allocated_octets.push( + datastore + .allocate_sled_underlay_subnet_octets( + &opctx, + rack_id, + Uuid::new_v4(), + ) + .await + .unwrap() + .subnet_octet, + ); + } + + let expected = vec![33, 34, 35, 36, 37]; + assert_eq!(expected, allocated_octets); + + // We should have 5 allocations in the DB, sorted appropriately + let allocations = + datastore.rack_subnet_allocations(&opctx, rack_id).await.unwrap(); + assert_eq!(5, allocations.len()); + assert_eq!( + expected, + allocations.iter().map(|a| a.subnet_octet).collect::>() + ); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } } diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 1c2e49e260..984ece2d0c 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -10,7 +10,7 @@ use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; use gateway_client::types::SpType; -use ipnetwork::IpNetwork; +use ipnetwork::{IpNetwork, Ipv6Network}; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; @@ -29,13 +29,14 @@ use nexus_types::external_api::params::{ AddressLotCreate, LoopbackAddressCreate, Route, SiloCreate, SwitchPortSettingsCreate, }; +use nexus_types::external_api::shared::Baseboard; use nexus_types::external_api::shared::FleetRole; use nexus_types::external_api::shared::SiloIdentityMode; use nexus_types::external_api::shared::SiloRole; +use nexus_types::external_api::shared::UninitializedSled; use nexus_types::external_api::views; -use nexus_types::external_api::views::Baseboard; -use nexus_types::external_api::views::UninitializedSled; use nexus_types::internal_api::params::DnsRecord; +use omicron_common::address::{get_64_subnet, Ipv6Subnet, RACK_PREFIX}; use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; @@ -45,7 +46,10 @@ use omicron_common::api::external::LookupResult; use omicron_common::api::external::Name; use omicron_common::api::external::NameOrId; use omicron_common::api::internal::shared::ExternalPortDiscovery; +use sled_agent_client::types::AddSledRequest; use sled_agent_client::types::EarlyNetworkConfigBody; +use sled_agent_client::types::StartSledAgentRequest; +use sled_agent_client::types::StartSledAgentRequestBody; use sled_agent_client::types::{ BgpConfig, BgpPeerConfig, EarlyNetworkConfig, PortConfigV1, RackNetworkConfigV1, RouteConfig as SledRouteConfig, @@ -584,20 +588,7 @@ impl super::Nexus { if rack.rack_subnet.is_some() { return Ok(()); } - let addr = self - .sled_list(opctx, &DataPageParams::max_page()) - .await? - .get(0) - .ok_or(Error::InternalError { - internal_message: "no sleds at time of bootstore sync".into(), - })? - .address(); - - let sa = sled_agent_client::Client::new( - &format!("http://{}", addr), - self.log.clone(), - ); - + let sa = self.get_any_sled_agent(opctx).await?; let result = sa .read_network_bootstore_config_cache() .await @@ -619,7 +610,7 @@ impl super::Nexus { opctx: &OpContext, ) -> Result { let rack = self.rack_lookup(opctx, &self.rack_id).await?; - let subnet = rack.subnet()?; + let subnet = rack_subnet(rack.rack_subnet)?; let db_ports = self.active_port_settings(opctx).await?; let mut ports = Vec::new(); @@ -726,18 +717,28 @@ impl super::Nexus { &self, opctx: &OpContext, ) -> ListResultVec { + debug!(self.log, "Getting latest collection"); // Grab the SPs from the last collection let limit = NonZeroU32::new(50).unwrap(); let collection = self .db_datastore .inventory_get_latest_collection(opctx, limit) .await?; + + // There can't be any uninitialized sleds we know about + // if there is no inventory. + let Some(collection) = collection else { + return Ok(vec![]); + }; + let pagparams = DataPageParams { marker: None, direction: dropshot::PaginationOrder::Descending, // TODO: This limit is only suitable for a single sled cluster limit: NonZeroU32::new(32).unwrap(), }; + + debug!(self.log, "Listing sleds"); let sleds = self.db_datastore.sled_list(opctx, &pagparams).await?; let mut uninitialized_sleds: Vec = collection @@ -767,4 +768,106 @@ impl super::Nexus { uninitialized_sleds.retain(|s| !sled_baseboards.contains(&s.baseboard)); Ok(uninitialized_sleds) } + + /// Add a sled to an intialized rack + pub(crate) async fn add_sled_to_initialized_rack( + &self, + opctx: &OpContext, + sled: UninitializedSled, + ) -> Result<(), Error> { + let baseboard_id = sled.baseboard.clone().into(); + let hw_baseboard_id = + self.db_datastore.find_hw_baseboard_id(opctx, baseboard_id).await?; + + let subnet = self.db_datastore.rack_subnet(opctx, sled.rack_id).await?; + let rack_subnet = + Ipv6Subnet::::from(rack_subnet(Some(subnet))?); + + let allocation = self + .db_datastore + .allocate_sled_underlay_subnet_octets( + opctx, + sled.rack_id, + hw_baseboard_id, + ) + .await?; + + // Convert the baseboard as necessary + let baseboard = sled_agent_client::types::Baseboard::Gimlet { + identifier: sled.baseboard.serial.clone(), + model: sled.baseboard.part.clone(), + revision: sled.baseboard.revision, + }; + + // Make the call to sled-agent + let req = AddSledRequest { + sled_id: baseboard, + start_request: StartSledAgentRequest { + generation: 0, + schema_version: 1, + body: StartSledAgentRequestBody { + id: allocation.sled_id, + rack_id: allocation.rack_id, + use_trust_quorum: true, + is_lrtq_learner: true, + subnet: sled_agent_client::types::Ipv6Subnet { + net: get_64_subnet( + rack_subnet, + allocation.subnet_octet.try_into().unwrap(), + ) + .net() + .into(), + }, + }, + }, + }; + let sa = self.get_any_sled_agent(opctx).await?; + sa.add_sled_to_initialized_rack(&req).await.map_err(|e| { + Error::InternalError { + internal_message: format!( + "failed to add sled with baseboard {:?} to rack {}: {e}", + sled.baseboard, allocation.rack_id + ), + } + })?; + + Ok(()) + } + + async fn get_any_sled_agent( + &self, + opctx: &OpContext, + ) -> Result { + let addr = self + .sled_list(opctx, &DataPageParams::max_page()) + .await? + .get(0) + .ok_or(Error::InternalError { + internal_message: "no sled agents available".into(), + })? + .address(); + + Ok(sled_agent_client::Client::new( + &format!("http://{}", addr), + self.log.clone(), + )) + } +} + +pub fn rack_subnet( + rack_subnet: Option, +) -> Result { + match rack_subnet { + Some(IpNetwork::V6(subnet)) => Ok(subnet), + Some(IpNetwork::V4(_)) => { + return Err(Error::InternalError { + internal_message: "rack subnet not IPv6".into(), + }) + } + None => { + return Err(Error::InternalError { + internal_message: "rack subnet not set".into(), + }) + } + } } diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index 8189c0a93d..c2931f1441 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -38,6 +38,9 @@ impl super::Nexus { // TODO-robustness we should have a limit on how many sled agents there can // be (for graceful degradation at large scale). + // + // TODO-multisled: This should not use the rack_id for the given nexus, + // unless the DNS lookups at sled-agent are only for rack-local nexuses. pub(crate) async fn upsert_sled( &self, opctx: &OpContext, diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 428632bcf5..78f675c28a 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -6,10 +6,11 @@ use super::{ console_api, device_auth, params, + shared::UninitializedSled, views::{ self, Certificate, Group, IdentityProvider, Image, IpPool, IpPoolRange, - PhysicalDisk, Project, Rack, Role, Silo, Sled, Snapshot, SshKey, - UninitializedSled, User, UserBuiltin, Vpc, VpcRouter, VpcSubnet, + PhysicalDisk, Project, Rack, Role, Silo, Sled, Snapshot, SshKey, User, + UserBuiltin, Vpc, VpcRouter, VpcSubnet, }, }; use crate::external_api::shared; @@ -223,6 +224,7 @@ pub(crate) fn external_api() -> NexusApiDescription { api.register(switch_list)?; api.register(switch_view)?; api.register(uninitialized_sled_list)?; + api.register(add_sled_to_initialized_rack)?; api.register(user_builtin_list)?; api.register(user_builtin_view)?; @@ -4402,6 +4404,31 @@ async fn uninitialized_sled_list( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } +/// Add a sled to an initialized rack +// +// TODO: In the future this should really be a PUT request, once we resolve +// https://github.com/oxidecomputer/omicron/issues/4494. It should also +// explicitly be tied to a rack via a `rack_id` path param. For now we assume +// we are only operating on single rack systems. +#[endpoint { + method = POST, + path = "/v1/system/hardware/sleds/", + tags = ["system/hardware"] +}] +async fn add_sled_to_initialized_rack( + rqctx: RequestContext>, + sled: TypedBody, +) -> Result { + let apictx = rqctx.context(); + let nexus = &apictx.nexus; + let handler = async { + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + nexus.add_sled_to_initialized_rack(&opctx, sled.into_inner()).await?; + Ok(HttpResponseUpdatedNoContent()) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + // Sleds /// List sleds diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 64790c49c2..5dfdcc151d 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -21,8 +21,10 @@ use nexus_test_utils::SLED_AGENT_UUID; use nexus_test_utils::SWITCH_UUID; use nexus_types::external_api::params; use nexus_types::external_api::shared; +use nexus_types::external_api::shared::Baseboard; use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::shared::Ipv4Range; +use nexus_types::external_api::shared::UninitializedSled; use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::ByteCount; use omicron_common::api::external::IdentityMetadataCreateParams; @@ -39,6 +41,7 @@ use omicron_test_utils::certificates::CertificateChain; use std::net::IpAddr; use std::net::Ipv4Addr; use std::str::FromStr; +use uuid::Uuid; lazy_static! { pub static ref HARDWARE_RACK_URL: String = @@ -57,6 +60,16 @@ lazy_static! { pub static ref SLED_INSTANCES_URL: String = format!("/v1/system/hardware/sleds/{}/instances", SLED_AGENT_UUID); + pub static ref DEMO_UNINITIALIZED_SLED: UninitializedSled = UninitializedSled { + baseboard: Baseboard { + serial: "demo-serial".to_string(), + part: "demo-part".to_string(), + revision: 6 + }, + rack_id: Uuid::new_v4(), + cubby: 1 + }; + // Global policy pub static ref SYSTEM_POLICY_URL: &'static str = "/v1/system/policy"; @@ -1577,7 +1590,9 @@ lazy_static! { url: "/v1/system/hardware/sleds", visibility: Visibility::Public, unprivileged_access: UnprivilegedAccess::None, - allowed_methods: vec![AllowedMethod::Get], + allowed_methods: vec![AllowedMethod::Get, AllowedMethod::Post( + serde_json::to_value(&*DEMO_UNINITIALIZED_SLED).unwrap() + )], }, VerifyEndpoint { diff --git a/nexus/tests/integration_tests/rack.rs b/nexus/tests/integration_tests/rack.rs index 2c191f27ae..9f77223871 100644 --- a/nexus/tests/integration_tests/rack.rs +++ b/nexus/tests/integration_tests/rack.rs @@ -10,8 +10,14 @@ use nexus_test_utils::http_testing::RequestBuilder; use nexus_test_utils::TEST_SUITE_PASSWORD; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; +use nexus_types::external_api::shared::UninitializedSled; use nexus_types::external_api::views::Rack; +use nexus_types::internal_api::params::Baseboard; +use nexus_types::internal_api::params::SledAgentStartupInfo; +use nexus_types::internal_api::params::SledRole; +use omicron_common::api::external::ByteCount; use omicron_nexus::TestInterfaces; +use uuid::Uuid; type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; @@ -77,3 +83,63 @@ async fn test_rack_initialization(cptestctx: &ControlPlaneTestContext) { ) .await; } + +#[nexus_test] +async fn test_uninitialized_sled_list(cptestctx: &ControlPlaneTestContext) { + let internal_client = &cptestctx.internal_client; + let external_client = &cptestctx.external_client; + let list_url = "/v1/system/hardware/uninitialized-sleds"; + let mut uninitialized_sleds = + NexusRequest::object_get(external_client, &list_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to get uninitialized sleds") + .parsed_body::>() + .unwrap(); + debug!(cptestctx.logctx.log, "{:#?}", uninitialized_sleds); + + // There are currently two fake sim gimlets created in the latest inventory + // collection as part of test setup. + assert_eq!(2, uninitialized_sleds.len()); + + // Insert one of these fake sleds into the `sled` table. + // Just pick some random fields other than `baseboard` + let baseboard = uninitialized_sleds.pop().unwrap().baseboard; + let sled_uuid = Uuid::new_v4(); + let sa = SledAgentStartupInfo { + sa_address: "[fd00:1122:3344:01::1]:8080".parse().unwrap(), + role: SledRole::Gimlet, + baseboard: Baseboard { + serial_number: baseboard.serial, + part_number: baseboard.part, + revision: baseboard.revision, + }, + usable_hardware_threads: 32, + usable_physical_ram: ByteCount::from_gibibytes_u32(100), + reservoir_size: ByteCount::from_mebibytes_u32(100), + }; + internal_client + .make_request( + Method::POST, + format!("/sled-agents/{sled_uuid}").as_str(), + Some(&sa), + StatusCode::NO_CONTENT, + ) + .await + .unwrap(); + + // Ensure there's only one unintialized sled remaining, and it's not + // the one that was just added into the `sled` table + let uninitialized_sleds_2 = + NexusRequest::object_get(external_client, &list_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to get uninitialized sleds") + .parsed_body::>() + .unwrap(); + debug!(cptestctx.logctx.log, "{:#?}", uninitialized_sleds); + assert_eq!(1, uninitialized_sleds_2.len()); + assert_eq!(uninitialized_sleds, uninitialized_sleds_2); +} diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 7f0c30c471..dd387ab979 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -110,6 +110,7 @@ snapshot_view GET /v1/snapshots/{snapshot} API operations found with tag "system/hardware" OPERATION ID METHOD URL PATH +add_sled_to_initialized_rack POST /v1/system/hardware/sleds networking_switch_port_apply_settings POST /v1/system/hardware/switch-port/{port}/settings networking_switch_port_clear_settings DELETE /v1/system/hardware/switch-port/{port}/settings networking_switch_port_list GET /v1/system/hardware/switch-port diff --git a/nexus/types/src/external_api/shared.rs b/nexus/types/src/external_api/shared.rs index 48fbb9c10d..a4c5ae1e62 100644 --- a/nexus/types/src/external_api/shared.rs +++ b/nexus/types/src/external_api/shared.rs @@ -245,6 +245,42 @@ pub enum UpdateableComponentType { HostOmicron, } +/// Properties that uniquely identify an Oxide hardware component +#[derive( + Clone, + Debug, + Serialize, + Deserialize, + JsonSchema, + PartialOrd, + Ord, + PartialEq, + Eq, +)] +pub struct Baseboard { + pub serial: String, + pub part: String, + pub revision: i64, +} + +/// A sled that has not been added to an initialized rack yet +#[derive( + Clone, + Debug, + Serialize, + Deserialize, + JsonSchema, + PartialOrd, + Ord, + PartialEq, + Eq, +)] +pub struct UninitializedSled { + pub baseboard: Baseboard, + pub rack_id: Uuid, + pub cubby: u16, +} + #[cfg(test)] mod test { use super::Policy; diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index b34fc7a542..9dfe36d63b 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -5,7 +5,7 @@ //! Views are response bodies, most of which are public lenses onto DB models. use crate::external_api::shared::{ - self, IpKind, IpRange, ServiceUsingCertificate, + self, Baseboard, IpKind, IpRange, ServiceUsingCertificate, }; use crate::identity::AssetIdentityMetadata; use api_identity::ObjectIdentity; @@ -274,44 +274,8 @@ pub struct Rack { pub identity: AssetIdentityMetadata, } -/// View of a sled that has not been added to an initialized rack yet -#[derive( - Clone, - Debug, - Serialize, - Deserialize, - JsonSchema, - PartialOrd, - Ord, - PartialEq, - Eq, -)] -pub struct UninitializedSled { - pub baseboard: Baseboard, - pub rack_id: Uuid, - pub cubby: u16, -} - // FRUs -/// Properties that uniquely identify an Oxide hardware component -#[derive( - Clone, - Debug, - Serialize, - Deserialize, - JsonSchema, - PartialOrd, - Ord, - PartialEq, - Eq, -)] -pub struct Baseboard { - pub serial: String, - pub part: String, - pub revision: i64, -} - // SLEDS /// An operator's view of a Sled. diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index c0991ebb17..bc25e8d4bd 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -25,7 +25,7 @@ use uuid::Uuid; /// /// Note that this may change if the sled is physically moved /// within the rack. -#[derive(Serialize, Deserialize, JsonSchema)] +#[derive(Serialize, Deserialize, JsonSchema, Debug)] #[serde(rename_all = "snake_case")] pub enum SledRole { /// The sled is a general compute sled. @@ -45,7 +45,7 @@ pub struct Baseboard { } /// Sent by a sled agent on startup to Nexus to request further instruction -#[derive(Serialize, Deserialize, JsonSchema)] +#[derive(Serialize, Deserialize, Debug, JsonSchema)] pub struct SledAgentStartupInfo { /// The address of the sled agent's API endpoint pub sa_address: SocketAddrV6, diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 19c323d894..9401727162 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -20,6 +20,8 @@ use std::sync::Arc; use strum::EnumIter; use uuid::Uuid; +use crate::external_api::shared::Baseboard; + /// Results of collecting hardware/software inventory from various Omicron /// components /// @@ -131,6 +133,12 @@ pub struct BaseboardId { pub serial_number: String, } +impl From for BaseboardId { + fn from(value: Baseboard) -> Self { + BaseboardId { part_number: value.part, serial_number: value.serial } + } +} + /// Caboose contents found during a collection /// /// These are normalized in the database. Each distinct `Caboose` is assigned a diff --git a/openapi/nexus.json b/openapi/nexus.json index 0d19e81d9a..704aa393db 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -3610,6 +3610,34 @@ "x-dropshot-pagination": { "required": [] } + }, + "post": { + "tags": [ + "system/hardware" + ], + "summary": "Add a sled to an initialized rack", + "operationId": "add_sled_to_initialized_rack", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UninitializedSled" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } } }, "/v1/system/hardware/sleds/{sled_id}": { @@ -13971,7 +13999,7 @@ ] }, "UninitializedSled": { - "description": "View of a sled that has not been added to an initialized rack yet", + "description": "A sled that has not been added to an initialized rack yet", "type": "object", "properties": { "baseboard": { diff --git a/schema/crdb/14.0.0/up1.sql b/schema/crdb/14.0.0/up1.sql new file mode 100644 index 0000000000..3bff831ceb --- /dev/null +++ b/schema/crdb/14.0.0/up1.sql @@ -0,0 +1,37 @@ +-- Table of all sled subnets allocated for sleds added to an already initialized +-- rack. The sleds in this table and their allocated subnets are created before +-- a sled is added to the `sled` table. Addition to the `sled` table occurs +-- after the sled is initialized and notifies Nexus about itself. +-- +-- For simplicity and space savings, this table doesn't actually contain the +-- full subnets for a given sled, but only the octet that extends a /56 rack +-- subnet to a /64 sled subnet. The rack subnet is maintained in the `rack` +-- table. +-- +-- This table does not include subnet octets allocated during RSS and therefore +-- all of the octets start at 33. This makes the data in this table purely additive +-- post-RSS, which also implies that we cannot re-use subnet octets if an original +-- sled that was part of RSS was removed from the cluster. +CREATE TABLE IF NOT EXISTS omicron.public.sled_underlay_subnet_allocation ( + -- The physical identity of the sled + -- (foreign key into `hw_baseboard_id` table) + hw_baseboard_id UUID PRIMARY KEY, + + -- The rack to which a sled is being added + -- (foreign key into `rack` table) + -- + -- We require this because the sled is not yet part of the sled table when + -- we first allocate a subnet for it. + rack_id UUID NOT NULL, + + -- The sled to which a subnet is being allocated + -- + -- Eventually will be a foreign key into the `sled` table when the sled notifies nexus + -- about itself after initialization. + sled_id UUID NOT NULL, + + -- The octet that extends a /56 rack subnet to a /64 sled subnet + -- + -- Always between 33 and 255 inclusive + subnet_octet INT2 NOT NULL UNIQUE CHECK (subnet_octet BETWEEN 33 AND 255) +); diff --git a/schema/crdb/14.0.0/up2.sql b/schema/crdb/14.0.0/up2.sql new file mode 100644 index 0000000000..c3e18fa166 --- /dev/null +++ b/schema/crdb/14.0.0/up2.sql @@ -0,0 +1,5 @@ +-- Add an index which allows pagination by {rack_id, sled_id} pairs. +CREATE UNIQUE INDEX IF NOT EXISTS lookup_subnet_allocation_by_rack_and_sled ON omicron.public.sled_underlay_subnet_allocation ( + rack_id, + sled_id +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index fc3bc37fd7..728b084982 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -158,6 +158,51 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_resource_by_sled ON omicron.public.sled id ); + +-- Table of all sled subnets allocated for sleds added to an already initialized +-- rack. The sleds in this table and their allocated subnets are created before +-- a sled is added to the `sled` table. Addition to the `sled` table occurs +-- after the sled is initialized and notifies Nexus about itself. +-- +-- For simplicity and space savings, this table doesn't actually contain the +-- full subnets for a given sled, but only the octet that extends a /56 rack +-- subnet to a /64 sled subnet. The rack subnet is maintained in the `rack` +-- table. +-- +-- This table does not include subnet octets allocated during RSS and therefore +-- all of the octets start at 33. This makes the data in this table purely additive +-- post-RSS, which also implies that we cannot re-use subnet octets if an original +-- sled that was part of RSS was removed from the cluster. +CREATE TABLE IF NOT EXISTS omicron.public.sled_underlay_subnet_allocation ( + -- The physical identity of the sled + -- (foreign key into `hw_baseboard_id` table) + hw_baseboard_id UUID PRIMARY KEY, + + -- The rack to which a sled is being added + -- (foreign key into `rack` table) + -- + -- We require this because the sled is not yet part of the sled table when + -- we first allocate a subnet for it. + rack_id UUID NOT NULL, + + -- The sled to which a subnet is being allocated + -- + -- Eventually will be a foreign key into the `sled` table when the sled notifies nexus + -- about itself after initialization. + sled_id UUID NOT NULL, + + -- The octet that extends a /56 rack subnet to a /64 sled subnet + -- + -- Always between 33 and 255 inclusive + subnet_octet INT2 NOT NULL UNIQUE CHECK (subnet_octet BETWEEN 33 AND 255) +); + +-- Add an index which allows pagination by {rack_id, sled_id} pairs. +CREATE UNIQUE INDEX IF NOT EXISTS lookup_subnet_allocation_by_rack_and_sled ON omicron.public.sled_underlay_subnet_allocation ( + rack_id, + sled_id +); + /* * Switches */ @@ -2952,7 +2997,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '13.0.0', NULL) + ( TRUE, NOW(), NOW(), '14.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT;