diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs index 5a9d6af810..0b766f9e9b 100644 --- a/nexus/db-model/src/deployment.rs +++ b/nexus/db-model/src/deployment.rs @@ -9,11 +9,13 @@ use crate::inventory::ZoneType; use crate::omicron_zone_config::{OmicronZone, OmicronZoneNic}; use crate::schema::{ blueprint, bp_omicron_physical_disk, bp_omicron_zone, bp_omicron_zone_nic, - bp_sled_omicron_physical_disks, bp_sled_omicron_zones, bp_target, + bp_sled_omicron_physical_disks, bp_sled_omicron_zones, bp_sled_state, + bp_target, }; use crate::typed_uuid::DbTypedUuid; use crate::{ - impl_enum_type, ipv6, Generation, MacAddr, Name, SqlU16, SqlU32, SqlU8, + impl_enum_type, ipv6, Generation, MacAddr, Name, SledState, SqlU16, SqlU32, + SqlU8, }; use chrono::{DateTime, Utc}; use ipnetwork::IpNetwork; @@ -103,6 +105,15 @@ impl From for nexus_types::deployment::BlueprintTarget { } } +/// See [`nexus_types::deployment::Blueprint::sled_state`]. +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = bp_sled_state)] +pub struct BpSledState { + pub blueprint_id: Uuid, + pub sled_id: DbTypedUuid, + pub sled_state: SledState, +} + /// See [`nexus_types::deployment::BlueprintPhysicalDisksConfig`]. #[derive(Queryable, Clone, Debug, Selectable, Insertable)] #[diesel(table_name = bp_sled_omicron_physical_disks)] diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 979395d907..3d16b978f6 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1504,6 +1504,15 @@ table! { } } +table! { + bp_sled_state (blueprint_id, sled_id) { + blueprint_id -> Uuid, + sled_id -> Uuid, + + sled_state -> crate::SledStateEnum, + } +} + table! { bp_sled_omicron_physical_disks (blueprint_id, sled_id) { blueprint_id -> Uuid, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index a86d030e48..5a263ea536 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(60, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(61, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(61, "blueprint-add-sled-state"), KnownVersion::new(60, "add-lookup-vmm-by-sled-id-index"), KnownVersion::new(59, "enforce-first-as-default"), KnownVersion::new(58, "insert-default-allowlist"), diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 97f7296368..6fdeed9ee5 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -40,12 +40,14 @@ use nexus_db_model::BpOmicronZone; use nexus_db_model::BpOmicronZoneNic; use nexus_db_model::BpSledOmicronPhysicalDisks; use nexus_db_model::BpSledOmicronZones; +use nexus_db_model::BpSledState; use nexus_db_model::BpTarget; use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintMetadata; use nexus_types::deployment::BlueprintPhysicalDisksConfig; use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::BlueprintZonesConfig; +use nexus_types::external_api::views::SledState; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; @@ -109,6 +111,16 @@ impl DataStore { let row_blueprint = DbBlueprint::from(blueprint); let blueprint_id = row_blueprint.id; + let sled_states = blueprint + .sled_state + .iter() + .map(|(&sled_id, &state)| BpSledState { + blueprint_id, + sled_id: sled_id.into(), + sled_state: state.into(), + }) + .collect::>(); + let sled_omicron_physical_disks = blueprint .blueprint_disks .iter() @@ -187,6 +199,16 @@ impl DataStore { .await?; } + // Insert all the sled states for this blueprint. + { + use db::schema::bp_sled_state::dsl as sled_state; + + let _ = diesel::insert_into(sled_state::bp_sled_state) + .values(sled_states) + .execute_async(&conn) + .await?; + } + // Insert all physical disks for this blueprint. { @@ -290,6 +312,41 @@ impl DataStore { ) }; + // Load the sled states for this blueprint. + let sled_state: BTreeMap = { + use db::schema::bp_sled_state::dsl; + + let mut sled_state = BTreeMap::new(); + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + while let Some(p) = paginator.next() { + let batch = paginated( + dsl::bp_sled_state, + dsl::sled_id, + &p.current_pagparams(), + ) + .filter(dsl::blueprint_id.eq(blueprint_id)) + .select(BpSledState::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + paginator = p.found_batch(&batch, &|s| s.sled_id); + + for s in batch { + let old = sled_state + .insert(s.sled_id.into(), s.sled_state.into()); + bail_unless!( + old.is_none(), + "found duplicate sled ID in bp_sled_state: {}", + s.sled_id + ); + } + } + sled_state + }; + // Read this blueprint's `bp_sled_omicron_zones` rows, which describes // the `OmicronZonesConfig` generation number for each sled that is a // part of this blueprint. Construct the BTreeMap we ultimately need, @@ -550,6 +607,7 @@ impl DataStore { id: blueprint_id, blueprint_zones, blueprint_disks, + sled_state, parent_blueprint_id, internal_dns_version, external_dns_version, @@ -578,6 +636,7 @@ impl DataStore { let ( nblueprints, + nsled_states, nsled_physical_disks, nphysical_disks, nsled_agent_zones, @@ -617,6 +676,17 @@ impl DataStore { )); } + // Remove rows associated with sled states. + let nsled_states = { + use db::schema::bp_sled_state::dsl; + diesel::delete( + dsl::bp_sled_state + .filter(dsl::blueprint_id.eq(blueprint_id)), + ) + .execute_async(&conn) + .await? + }; + // Remove rows associated with Omicron physical disks let nsled_physical_disks = { use db::schema::bp_sled_omicron_physical_disks::dsl; @@ -670,6 +740,7 @@ impl DataStore { Ok(( nblueprints, + nsled_states, nsled_physical_disks, nphysical_disks, nsled_agent_zones, @@ -688,6 +759,7 @@ impl DataStore { info!(&opctx.log, "removed blueprint"; "blueprint_id" => blueprint_id.to_string(), "nblueprints" => nblueprints, + "nsled_states" => nsled_states, "nsled_physical_disks" => nsled_physical_disks, "nphysical_disks" => nphysical_disks, "nsled_agent_zones" => nsled_agent_zones, @@ -1267,7 +1339,6 @@ mod tests { use nexus_types::external_api::views::PhysicalDiskPolicy; use nexus_types::external_api::views::PhysicalDiskState; use nexus_types::external_api::views::SledPolicy; - use nexus_types::external_api::views::SledState; use nexus_types::inventory::Collection; use omicron_common::address::Ipv6Subnet; use omicron_common::disk::DiskIdentity; diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index a072d65147..8e8913f7bd 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -984,6 +984,7 @@ mod test { BlueprintZoneDisposition, OmicronZoneExternalSnatIp, }; use nexus_types::external_api::shared::SiloIdentityMode; + use nexus_types::external_api::views::SledState; use nexus_types::identity::Asset; use nexus_types::internal_api::params::DnsRecord; use nexus_types::inventory::NetworkInterface; @@ -1016,6 +1017,7 @@ mod test { id: Uuid::new_v4(), blueprint_zones: BTreeMap::new(), blueprint_disks: BTreeMap::new(), + sled_state: BTreeMap::new(), parent_blueprint_id: None, internal_dns_version: *Generation::new(), external_dns_version: *Generation::new(), @@ -1255,6 +1257,12 @@ mod test { } } + fn sled_states_active( + sled_ids: impl Iterator, + ) -> BTreeMap { + sled_ids.map(|sled_id| (sled_id, SledState::Active)).collect() + } + #[tokio::test] async fn rack_set_initialized_with_services() { let test_name = "rack_set_initialized_with_services"; @@ -1487,6 +1495,7 @@ mod test { } let blueprint = Blueprint { id: Uuid::new_v4(), + sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), parent_blueprint_id: None, @@ -1742,6 +1751,7 @@ mod test { } let blueprint = Blueprint { id: Uuid::new_v4(), + sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), parent_blueprint_id: None, @@ -1953,6 +1963,7 @@ mod test { } let blueprint = Blueprint { id: Uuid::new_v4(), + sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), parent_blueprint_id: None, @@ -2091,6 +2102,7 @@ mod test { } let blueprint = Blueprint { id: Uuid::new_v4(), + sled_state: sled_states_active(blueprint_zones.keys().copied()), blueprint_zones, blueprint_disks: BTreeMap::new(), parent_blueprint_id: None, diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 632ea501e3..91843abf2e 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -1233,35 +1233,24 @@ mod tests { use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; use crate::db::model::Project; use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; - use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; + use nexus_db_model::IncompleteNetworkInterface; use nexus_db_model::SledUpdate; + use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; + use nexus_reconfigurator_planning::system::SledBuilder; + use nexus_reconfigurator_planning::system::SystemDescription; use nexus_test_utils::db::test_setup_database; - use nexus_types::deployment::blueprint_zone_type; use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; - use nexus_types::deployment::BlueprintZoneType; - use nexus_types::deployment::BlueprintZonesConfig; - use nexus_types::deployment::OmicronZoneExternalFloatingIp; use nexus_types::external_api::params; use nexus_types::identity::Asset; - use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; use omicron_common::api::external; use omicron_common::api::external::Generation; - use omicron_common::api::external::IpNet; - use omicron_common::api::external::MacAddr; - use omicron_common::api::external::Vni; - use omicron_common::api::internal::shared::NetworkInterface; - use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_test_utils::dev; - use omicron_uuid_kinds::ExternalIpUuid; use omicron_uuid_kinds::GenericUuid; - use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; use slog::info; - use std::collections::BTreeMap; - use std::net::IpAddr; // Test that we detect the right error condition and return None when we // fail to insert a VPC due to VNI exhaustion. @@ -1481,128 +1470,6 @@ mod tests { logctx.cleanup_successful(); } - #[derive(Debug)] - struct Harness { - rack_id: Uuid, - sled_ids: Vec, - nexuses: Vec, - } - - #[derive(Debug)] - struct HarnessNexus { - sled_id: SledUuid, - id: OmicronZoneUuid, - external_ip: OmicronZoneExternalFloatingIp, - mac: MacAddr, - nic_id: Uuid, - nic_ip: IpAddr, - } - - impl Harness { - fn new(num_sleds: usize) -> Self { - let mut sled_ids = - (0..num_sleds).map(|_| SledUuid::new_v4()).collect::>(); - sled_ids.sort(); - - // RFC 5737 TEST-NET-1 - let mut nexus_external_ips = - "192.0.2.0/24".parse::().unwrap().iter(); - let mut nexus_nic_ips = NEXUS_OPTE_IPV4_SUBNET - .iter() - .skip(NUM_INITIAL_RESERVED_IP_ADDRESSES) - .map(IpAddr::from); - let mut nexus_macs = MacAddr::iter_system(); - let nexuses = sled_ids - .iter() - .copied() - .map(|sled_id| HarnessNexus { - sled_id, - id: OmicronZoneUuid::new_v4(), - external_ip: OmicronZoneExternalFloatingIp { - id: ExternalIpUuid::new_v4(), - ip: nexus_external_ips.next().unwrap(), - }, - mac: nexus_macs.next().unwrap(), - nic_id: Uuid::new_v4(), - nic_ip: nexus_nic_ips.next().unwrap(), - }) - .collect::>(); - Self { rack_id: Uuid::new_v4(), sled_ids, nexuses } - } - - fn db_sleds(&self) -> impl Iterator + '_ { - self.sled_ids.iter().copied().map(|sled_id| { - SledUpdate::new( - sled_id.into_untyped_uuid(), - "[::1]:0".parse().unwrap(), - sled_baseboard_for_test(), - sled_system_hardware_for_test(), - self.rack_id, - Generation::new().into(), - ) - }) - } - - fn db_nics( - &self, - ) -> impl Iterator + '_ - { - self.nexuses.iter().map(|nexus| { - let name = format!("test-nexus-{}", nexus.id); - db::model::IncompleteNetworkInterface::new_service( - nexus.nic_id, - nexus.id.into_untyped_uuid(), - NEXUS_VPC_SUBNET.clone(), - IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: name, - }, - nexus.nic_ip, - nexus.mac, - 0, - ) - .expect("failed to create incomplete Nexus NIC") - }) - } - - fn blueprint_zone_configs( - &self, - ) -> impl Iterator + '_ - { - self.nexuses.iter().zip(self.db_nics()).map(|(nexus, nic)| { - let config = BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: nexus.id, - underlay_address: "::1".parse().unwrap(), - zone_type: BlueprintZoneType::Nexus( - blueprint_zone_type::Nexus { - internal_address: "[::1]:0".parse().unwrap(), - external_ip: nexus.external_ip, - nic: NetworkInterface { - id: nic.identity.id, - kind: NetworkInterfaceKind::Service { - id: nexus.id.into_untyped_uuid(), - }, - name: format!("test-nic-{}", nic.identity.id) - .parse() - .unwrap(), - ip: nic.ip.unwrap(), - mac: nic.mac.unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: nic.slot.unwrap(), - }, - external_tls: false, - external_dns_servers: Vec::new(), - }, - ), - }; - (nexus.sled_id, config) - }) - } - } - async fn assert_service_sled_ids( datastore: &DataStore, expected_sled_ids: &[SledUuid], @@ -1618,6 +1485,28 @@ mod tests { assert_eq!(expected_sled_ids, service_sled_ids); } + async fn bp_insert_and_make_target( + opctx: &OpContext, + datastore: &DataStore, + bp: &Blueprint, + ) { + datastore + .blueprint_insert(opctx, bp) + .await + .expect("inserted blueprint"); + datastore + .blueprint_target_set_current( + opctx, + BlueprintTarget { + target_id: bp.id, + enabled: true, + time_made_target: Utc::now(), + }, + ) + .await + .expect("made blueprint the target"); + } + #[tokio::test] async fn test_vpc_resolve_to_sleds_uses_current_target_blueprint() { // Test setup. @@ -1628,67 +1517,84 @@ mod tests { let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; - // Create five sleds. - let harness = Harness::new(5); - for sled in harness.db_sleds() { - datastore.sled_upsert(sled).await.expect("failed to upsert sled"); + // Set up our fake system with 5 sleds. + let rack_id = Uuid::new_v4(); + let mut system = SystemDescription::new(); + let mut sled_ids = Vec::new(); + for _ in 0..5 { + let sled_id = SledUuid::new_v4(); + sled_ids.push(sled_id); + system.sled(SledBuilder::new().id(sled_id)).expect("adding sled"); + datastore + .sled_upsert(SledUpdate::new( + sled_id.into_untyped_uuid(), + "[::1]:0".parse().unwrap(), + sled_baseboard_for_test(), + sled_system_hardware_for_test(), + rack_id, + Generation::new().into(), + )) + .await + .expect("upserting sled"); } - - // We don't have a blueprint yet, so we shouldn't find any services on - // sleds. - assert_service_sled_ids(&datastore, &[]).await; - - // Create a blueprint that has a Nexus on our third sled. (This - // blueprint is completely invalid in many ways, but all we care about - // here is inserting relevant records in `bp_omicron_zone`.) - let bp1_zones = { - let (sled_id, zone_config) = harness - .blueprint_zone_configs() - .nth(2) - .expect("fewer than 3 services in test harness"); - let mut zones = BTreeMap::new(); - zones.insert( - sled_id, - BlueprintZonesConfig { - generation: Generation::new(), - zones: vec![zone_config], + sled_ids.sort_unstable(); + let planning_input = system + .to_planning_input_builder() + .expect("creating planning builder") + .build(); + + // Helper to convert a zone's nic into an insertable nic. + let db_nic_from_zone = |zone_config: &BlueprintZoneConfig| { + let (_, nic) = zone_config + .zone_type + .external_networking() + .expect("external networking for zone type"); + IncompleteNetworkInterface::new_service( + nic.id, + zone_config.id.into_untyped_uuid(), + NEXUS_VPC_SUBNET.clone(), + IdentityMetadataCreateParams { + name: nic.name.clone(), + description: nic.name.to_string(), }, - ); - zones - }; - let bp1_id = Uuid::new_v4(); - let bp1 = Blueprint { - id: bp1_id, - blueprint_zones: bp1_zones, - blueprint_disks: BTreeMap::new(), - parent_blueprint_id: None, - internal_dns_version: Generation::new(), - external_dns_version: Generation::new(), - time_created: Utc::now(), - creator: "test".to_string(), - comment: "test".to_string(), + nic.ip, + nic.mac, + nic.slot, + ) + .expect("creating service nic") }; - datastore - .blueprint_insert(&opctx, &bp1) - .await - .expect("failed to insert blueprint"); - // We haven't set a blueprint target yet, so we should still fail to see - // any services on sleds. + // Create an initial, empty blueprint, and make it the target. + let bp0 = BlueprintBuilder::build_empty_with_sleds( + sled_ids.iter().copied(), + "test", + ); + bp_insert_and_make_target(&opctx, &datastore, &bp0).await; + + // Our blueprint doesn't describe any services, so we shouldn't find any + // sled IDs running services. assert_service_sled_ids(&datastore, &[]).await; - // Make bp1 the current target. - datastore - .blueprint_target_set_current( - &opctx, - BlueprintTarget { - target_id: bp1_id, - enabled: true, - time_made_target: Utc::now(), - }, + // Create a blueprint that has a Nexus on our third sled. + let bp1 = { + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &bp0, + &planning_input, + "test", ) - .await - .expect("failed to set blueprint target"); + .expect("created blueprint builder"); + builder + .sled_ensure_zone_multiple_nexus_with_config( + sled_ids[2], + 1, + false, + Vec::new(), + ) + .expect("added nexus to third sled"); + builder.build() + }; + bp_insert_and_make_target(&opctx, &datastore, &bp1).await; // bp1 is the target, but we haven't yet inserted a vNIC record, so // we still won't see any services on sleds. @@ -1696,182 +1602,105 @@ mod tests { // Insert the relevant service NIC record (normally performed by the // reconfigurator's executor). - datastore + let bp1_nic = datastore .service_create_network_interface_raw( &opctx, - harness.db_nics().nth(2).unwrap(), + db_nic_from_zone(&bp1.blueprint_zones[&sled_ids[2]].zones[0]), ) .await .expect("failed to insert service VNIC"); - // We should now see our third sled running a service. - assert_service_sled_ids(&datastore, &[harness.sled_ids[2]]).await; - - // Create another blueprint with no services and make it the target. - let bp2_id = Uuid::new_v4(); - let bp2 = Blueprint { - id: bp2_id, - blueprint_zones: BTreeMap::new(), - blueprint_disks: BTreeMap::new(), - parent_blueprint_id: Some(bp1_id), - internal_dns_version: Generation::new(), - external_dns_version: Generation::new(), - time_created: Utc::now(), - creator: "test".to_string(), - comment: "test".to_string(), + assert_service_sled_ids(&datastore, &[sled_ids[2]]).await; + + // Create another blueprint, remove the one nexus we added, and make it + // the target. + let bp2 = { + let mut bp2 = bp1.clone(); + bp2.id = Uuid::new_v4(); + bp2.parent_blueprint_id = Some(bp1.id); + let sled2_zones = bp2 + .blueprint_zones + .get_mut(&sled_ids[2]) + .expect("zones for third sled"); + sled2_zones.zones.clear(); + sled2_zones.generation = sled2_zones.generation.next(); + bp2 }; - datastore - .blueprint_insert(&opctx, &bp2) - .await - .expect("failed to insert blueprint"); - datastore - .blueprint_target_set_current( - &opctx, - BlueprintTarget { - target_id: bp2_id, - enabled: true, - time_made_target: Utc::now(), - }, - ) - .await - .expect("failed to set blueprint target"); + bp_insert_and_make_target(&opctx, &datastore, &bp2).await; // We haven't removed the service NIC record, but we should no longer // see the third sled here. We should be back to no sleds with services. assert_service_sled_ids(&datastore, &[]).await; - // Insert a service NIC record for our fourth sled's Nexus. This - // shouldn't change our VPC resolution. + // Delete the service NIC record so we can reuse this IP later. datastore - .service_create_network_interface_raw( + .service_delete_network_interface( &opctx, - harness.db_nics().nth(3).unwrap(), + bp1.blueprint_zones[&sled_ids[2]].zones[0] + .id + .into_untyped_uuid(), + bp1_nic.id(), ) .await - .expect("failed to insert service VNIC"); - assert_service_sled_ids(&datastore, &[]).await; - - // Create a blueprint that has a Nexus on our fourth sled. This - // shouldn't change our VPC resolution. - let bp3_zones = { - let (sled_id, zone_config) = harness - .blueprint_zone_configs() - .nth(3) - .expect("fewer than 3 services in test harness"); - let mut zones = BTreeMap::new(); - zones.insert( - sled_id, - BlueprintZonesConfig { - generation: Generation::new(), - zones: vec![zone_config], - }, - ); - zones - }; - let bp3_id = Uuid::new_v4(); - let bp3 = Blueprint { - id: bp3_id, - blueprint_zones: bp3_zones, - blueprint_disks: BTreeMap::new(), - parent_blueprint_id: Some(bp2_id), - internal_dns_version: Generation::new(), - external_dns_version: Generation::new(), - time_created: Utc::now(), - creator: "test".to_string(), - comment: "test".to_string(), + .expect("deleted bp1 nic"); + + // Create a blueprint with Nexus on all our sleds. + let bp3 = { + let mut builder = BlueprintBuilder::new_based_on( + &logctx.log, + &bp2, + &planning_input, + "test", + ) + .expect("created blueprint builder"); + for &sled_id in &sled_ids { + builder + .sled_ensure_zone_multiple_nexus_with_config( + sled_id, + 1, + false, + Vec::new(), + ) + .expect("added nexus to third sled"); + } + builder.build() }; - datastore - .blueprint_insert(&opctx, &bp3) - .await - .expect("failed to insert blueprint"); - assert_service_sled_ids(&datastore, &[]).await; - // Make this blueprint the target. We've already created the service - // VNIC, so we should immediately see our fourth sled in VPC resolution. - datastore - .blueprint_target_set_current( - &opctx, - BlueprintTarget { - target_id: bp3_id, - enabled: true, - time_made_target: Utc::now(), - }, - ) - .await - .expect("failed to set blueprint target"); - assert_service_sled_ids(&datastore, &[harness.sled_ids[3]]).await; + // Insert the service NIC records for all the Nexuses. + for &sled_id in &sled_ids { + datastore + .service_create_network_interface_raw( + &opctx, + db_nic_from_zone(&bp3.blueprint_zones[&sled_id].zones[0]), + ) + .await + .expect("failed to insert service VNIC"); + } - // --- + // We haven't made bp3 the target yet, so our resolution is still based + // on bp2; more service vNICs shouldn't matter. + assert_service_sled_ids(&datastore, &[]).await; - // Add a vNIC record for our fifth sled's Nexus, then create a blueprint - // that includes sleds with indexes 2, 3, and 4. Make it the target, - // and ensure we resolve to all five sleds. - datastore - .service_create_network_interface_raw( - &opctx, - harness.db_nics().nth(4).unwrap(), - ) - .await - .expect("failed to insert service VNIC"); - let bp4_zones = { - let mut zones = BTreeMap::new(); - for (sled_id, zone_config) in - harness.blueprint_zone_configs().skip(2) - { - zones.insert( - sled_id, - BlueprintZonesConfig { - generation: Generation::new(), - zones: vec![zone_config], - }, - ); - } - zones - }; - let bp4_id = Uuid::new_v4(); - let bp4 = Blueprint { - id: bp4_id, - blueprint_zones: bp4_zones, - blueprint_disks: BTreeMap::new(), - parent_blueprint_id: Some(bp3_id), - internal_dns_version: Generation::new(), - external_dns_version: Generation::new(), - time_created: Utc::now(), - creator: "test".to_string(), - comment: "test".to_string(), - }; - datastore - .blueprint_insert(&opctx, &bp4) - .await - .expect("failed to insert blueprint"); - datastore - .blueprint_target_set_current( - &opctx, - BlueprintTarget { - target_id: bp4_id, - enabled: true, - time_made_target: Utc::now(), - }, - ) - .await - .expect("failed to set blueprint target"); - assert_service_sled_ids(&datastore, &harness.sled_ids[2..]).await; + // Make bp3 the target; we should immediately resolve that there are + // services on the sleds we set up in bp3. + bp_insert_and_make_target(&opctx, &datastore, &bp3).await; + assert_service_sled_ids(&datastore, &sled_ids).await; // --- // Mark some sleds as ineligible. Only the non-provisionable and // in-service sleds should be returned. let ineligible = IneligibleSleds { - expunged: harness.sled_ids[0], - decommissioned: harness.sled_ids[1], - illegal_decommissioned: harness.sled_ids[2], - non_provisionable: harness.sled_ids[3], + expunged: sled_ids[0], + decommissioned: sled_ids[1], + illegal_decommissioned: sled_ids[2], + non_provisionable: sled_ids[3], }; ineligible .setup(&opctx, &datastore) .await .expect("failed to set up ineligible sleds"); - assert_service_sled_ids(&datastore, &harness.sled_ids[3..=4]).await; + assert_service_sled_ids(&datastore, &sled_ids[3..=4]).await; // --- @@ -1880,91 +1709,41 @@ mod tests { .undo(&opctx, &datastore) .await .expect("failed to undo ineligible sleds"); + assert_service_sled_ids(&datastore, &sled_ids).await; // Make a new blueprint marking one of the zones as quiesced and one as // expunged. Ensure that the sled with *quiesced* zone is returned by // vpc_resolve_to_sleds, but the sled with the *expunged* zone is not. // (But other services are still running.) - let bp5_zones = { - let mut zones = BTreeMap::new(); - // Skip over sled index 0 (should be excluded). - let mut iter = harness.blueprint_zone_configs().skip(1); - - // Sled index 1's zone is active (should be included). - let (sled_id, zone_config) = iter.next().unwrap(); - zones.insert( - sled_id, - BlueprintZonesConfig { - generation: Generation::new(), - zones: vec![zone_config], - }, - ); - - // We never created a vNIC record for sled 1; do so now. - datastore - .service_create_network_interface_raw( - &opctx, - harness.db_nics().nth(1).unwrap(), - ) - .await - .expect("failed to insert service VNIC"); - - // Sled index 2's zone is quiesced (should be included). - let (sled_id, mut zone_config) = iter.next().unwrap(); - zone_config.disposition = BlueprintZoneDisposition::Quiesced; - zones.insert( - sled_id, - BlueprintZonesConfig { - generation: Generation::new(), - zones: vec![zone_config], - }, - ); + let bp4 = { + let mut bp4 = bp3.clone(); + bp4.id = Uuid::new_v4(); + bp4.parent_blueprint_id = Some(bp3.id); + + // Sled index 2's Nexus is quiesced (should be included). + let sled2 = bp4 + .blueprint_zones + .get_mut(&sled_ids[2]) + .expect("zones for sled"); + sled2.zones[0].disposition = BlueprintZoneDisposition::Quiesced; + sled2.generation = sled2.generation.next(); // Sled index 3's zone is expunged (should be excluded). - let (sled_id, mut zone_config) = iter.next().unwrap(); - zone_config.disposition = BlueprintZoneDisposition::Expunged; - zones.insert( - sled_id, - BlueprintZonesConfig { - generation: Generation::new(), - zones: vec![zone_config], - }, - ); - - // Sled index 4's zone is not in the blueprint (should be excluded). - - zones + let sled3 = bp4 + .blueprint_zones + .get_mut(&sled_ids[3]) + .expect("zones for sled"); + sled3.zones[0].disposition = BlueprintZoneDisposition::Expunged; + sled3.generation = sled3.generation.next(); + + bp4 }; - - let bp5_id = Uuid::new_v4(); - let bp5 = Blueprint { - id: bp5_id, - blueprint_zones: bp5_zones, - blueprint_disks: BTreeMap::new(), - parent_blueprint_id: Some(bp4_id), - internal_dns_version: Generation::new(), - external_dns_version: Generation::new(), - time_created: Utc::now(), - creator: "test".to_string(), - comment: "test".to_string(), - }; - - datastore - .blueprint_insert(&opctx, &bp5) - .await - .expect("failed to insert blueprint"); - datastore - .blueprint_target_set_current( - &opctx, - BlueprintTarget { - target_id: bp5_id, - enabled: true, - time_made_target: Utc::now(), - }, - ) - .await - .expect("failed to set blueprint target"); - assert_service_sled_ids(&datastore, &harness.sled_ids[1..=2]).await; + bp_insert_and_make_target(&opctx, &datastore, &bp4).await; + assert_service_sled_ids( + &datastore, + &[sled_ids[0], sled_ids[1], sled_ids[2], sled_ids[4]], + ) + .await; db.cleanup().await.unwrap(); logctx.cleanup_successful(); diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index e0105f1f64..6a3c1755cf 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -480,6 +480,7 @@ mod test { use nexus_types::deployment::SledFilter; use nexus_types::external_api::params; use nexus_types::external_api::shared; + use nexus_types::external_api::views::SledState; use nexus_types::identity::Resource; use nexus_types::internal_api::params::DnsConfigParams; use nexus_types::internal_api::params::DnsConfigZone; @@ -557,6 +558,10 @@ mod test { // `BlueprintZonesConfig`. This is going to get more painful over time // as we add to blueprints, but for now we can make this work. let mut blueprint_zones = BTreeMap::new(); + + // Also assume any sled in the collection is active. + let mut sled_state = BTreeMap::new(); + for (sled_id, zones_config) in collection.omicron_zones { blueprint_zones.insert( sled_id, @@ -580,6 +585,7 @@ mod test { .collect(), }, ); + sled_state.insert(sled_id, SledState::Active); } let dns_empty = dns_config_empty(); @@ -589,6 +595,7 @@ mod test { id: Uuid::new_v4(), blueprint_zones, blueprint_disks: BTreeMap::new(), + sled_state, parent_blueprint_id: None, internal_dns_version: initial_dns_generation, external_dns_version: Generation::new(), diff --git a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs index a90b3c8e59..89287713c2 100644 --- a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs +++ b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs @@ -136,6 +136,7 @@ mod test { id, blueprint_zones: BTreeMap::new(), blueprint_disks, + sled_state: BTreeMap::new(), parent_blueprint_id: None, internal_dns_version: Generation::new(), external_dns_version: Generation::new(), diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs index da74023841..3248269175 100644 --- a/nexus/reconfigurator/execution/src/omicron_zones.rs +++ b/nexus/reconfigurator/execution/src/omicron_zones.rs @@ -126,6 +126,7 @@ mod test { id, blueprint_zones, blueprint_disks: BTreeMap::new(), + sled_state: BTreeMap::new(), parent_blueprint_id: None, internal_dns_version: Generation::new(), external_dns_version: Generation::new(), diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index dc8b1452a7..0d187af50c 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -30,6 +30,7 @@ use nexus_types::deployment::PlanningInput; use nexus_types::deployment::SledFilter; use nexus_types::deployment::SledResources; use nexus_types::deployment::ZpoolName; +use nexus_types::external_api::views::SledState; use omicron_common::address::get_internal_dns_server_addresses; use omicron_common::address::get_sled_address; use omicron_common::address::get_switch_zone_address; @@ -141,6 +142,7 @@ pub struct BlueprintBuilder<'a> { // corresponding fields in `Blueprint`. pub(super) zones: BlueprintZonesBuilder<'a>, disks: BlueprintDisksBuilder<'a>, + sled_state: BTreeMap, creator: String, comments: Vec, @@ -200,10 +202,16 @@ impl<'a> BlueprintBuilder<'a> { }) .collect::>(); let num_sleds = blueprint_zones.len(); + let sled_state = blueprint_zones + .keys() + .copied() + .map(|sled_id| (sled_id, SledState::Active)) + .collect(); Blueprint { id: rng.blueprint_rng.next(), blueprint_zones, blueprint_disks: BTreeMap::new(), + sled_state, parent_blueprint_id: None, internal_dns_version: Generation::new(), external_dns_version: Generation::new(), @@ -332,6 +340,21 @@ impl<'a> BlueprintBuilder<'a> { let available_system_macs = AvailableIterator::new(MacAddr::iter_system(), used_macs); + let sled_state = input + .all_sleds(SledFilter::All) + .map(|(sled_id, details)| { + // Prefer the sled state from our parent blueprint for sleds + // that were in it; there may be new sleds in `input`, in which + // case we'll use their current state as our starting point. + let state = parent_blueprint + .sled_state + .get(&sled_id) + .copied() + .unwrap_or(details.state); + (sled_id, state) + }) + .collect(); + Ok(BlueprintBuilder { log, parent_blueprint, @@ -339,6 +362,7 @@ impl<'a> BlueprintBuilder<'a> { sled_ip_allocators: BTreeMap::new(), zones: BlueprintZonesBuilder::new(parent_blueprint), disks: BlueprintDisksBuilder::new(parent_blueprint), + sled_state, creator: creator.to_owned(), comments: Vec::new(), nexus_v4_ips, @@ -362,6 +386,7 @@ impl<'a> BlueprintBuilder<'a> { id: self.rng.blueprint_rng.next(), blueprint_zones, blueprint_disks, + sled_state: self.sled_state, parent_blueprint_id: Some(self.parent_blueprint.id), internal_dns_version: self.input.internal_dns_version(), external_dns_version: self.input.external_dns_version(), diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs index 5a92d36895..1291e72a9b 100644 --- a/nexus/src/app/background/blueprint_execution.rs +++ b/nexus/src/app/background/blueprint_execution.rs @@ -125,6 +125,7 @@ mod test { BlueprintTarget, BlueprintZoneConfig, BlueprintZoneDisposition, BlueprintZoneType, BlueprintZonesConfig, }; + use nexus_types::external_api::views::SledState; use nexus_types::inventory::OmicronZoneDataset; use omicron_common::api::external::Generation; use omicron_uuid_kinds::GenericUuid; @@ -147,6 +148,12 @@ mod test { dns_version: Generation, ) -> (BlueprintTarget, Blueprint) { let id = Uuid::new_v4(); + // Assume all sleds are active. + let sled_state = blueprint_zones + .keys() + .copied() + .map(|sled_id| (sled_id, SledState::Active)) + .collect::>(); ( BlueprintTarget { target_id: id, @@ -157,6 +164,7 @@ mod test { id, blueprint_zones, blueprint_disks, + sled_state, parent_blueprint_id: None, internal_dns_version: dns_version, external_dns_version: dns_version, diff --git a/nexus/src/app/background/blueprint_load.rs b/nexus/src/app/background/blueprint_load.rs index 9c13d8e70a..8334abecb5 100644 --- a/nexus/src/app/background/blueprint_load.rs +++ b/nexus/src/app/background/blueprint_load.rs @@ -211,6 +211,7 @@ mod test { id, blueprint_zones: BTreeMap::new(), blueprint_disks: BTreeMap::new(), + sled_state: BTreeMap::new(), parent_blueprint_id: Some(parent_blueprint_id), internal_dns_version: Generation::new(), external_dns_version: Generation::new(), diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 3d0dfb0096..23d84ee702 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -33,6 +33,7 @@ use nexus_types::deployment::BlueprintZonesConfig; use nexus_types::deployment::OmicronZoneExternalFloatingAddr; use nexus_types::deployment::OmicronZoneExternalFloatingIp; use nexus_types::external_api::params::UserId; +use nexus_types::external_api::views::SledState; use nexus_types::internal_api::params::Certificate; use nexus_types::internal_api::params::DatasetCreateRequest; use nexus_types::internal_api::params::DatasetKind; @@ -754,18 +755,21 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { let blueprint = { let mut blueprint_zones = BTreeMap::new(); + let mut sled_state = BTreeMap::new(); for (maybe_sled_agent, zones) in [ (self.sled_agent.as_ref(), &self.blueprint_zones), (self.sled_agent2.as_ref(), &self.blueprint_zones2), ] { if let Some(sa) = maybe_sled_agent { + let sled_id = SledUuid::from_untyped_uuid(sa.sled_agent.id); blueprint_zones.insert( - SledUuid::from_untyped_uuid(sa.sled_agent.id), + sled_id, BlueprintZonesConfig { generation: Generation::new().next(), zones: zones.clone(), }, ); + sled_state.insert(sled_id, SledState::Active); } } Blueprint { @@ -776,6 +780,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { // // However, for now, this isn't necessary. blueprint_disks: BTreeMap::new(), + sled_state, parent_blueprint_id: None, internal_dns_version: dns_config .generation diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index b818b99794..fe013b8ca4 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -12,6 +12,7 @@ //! nexus/db-model, but nexus/reconfigurator/planning does not currently know //! about nexus/db-model and it's convenient to separate these concerns.) +use crate::external_api::views::SledState; use crate::internal_api::params::DnsConfigParams; use crate::inventory::Collection; pub use crate::inventory::OmicronZoneConfig; @@ -115,6 +116,9 @@ pub struct Blueprint { /// A map of sled id -> disks in use on each sled. pub blueprint_disks: BTreeMap, + /// A map of sled id -> desired state of the sled. + pub sled_state: BTreeMap, + /// which blueprint this blueprint is based on pub parent_blueprint_id: Option, diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 82027aa90e..a5e0d708eb 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -1785,6 +1785,13 @@ "type": "string", "format": "uuid" }, + "sled_state": { + "description": "A map of sled id -> desired state of the sled.", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/SledState" + } + }, "time_created": { "description": "when this blueprint was generated (for debugging)", "type": "string", @@ -1799,6 +1806,7 @@ "external_dns_version", "id", "internal_dns_version", + "sled_state", "time_created" ] }, @@ -4695,6 +4703,25 @@ "sled" ] }, + "SledState": { + "description": "The current state of the sled, as determined by Nexus.", + "oneOf": [ + { + "description": "The sled is currently active, and has resources allocated on it.", + "type": "string", + "enum": [ + "active" + ] + }, + { + "description": "The sled has been permanently removed from service.\n\nThis is a terminal state: once a particular sled ID is decommissioned, it will never return to service. (The actual hardware may be reused, but it will be treated as a brand-new sled.)", + "type": "string", + "enum": [ + "decommissioned" + ] + } + ] + }, "SourceNatConfig": { "description": "An IP address and port range used for source NAT, i.e., making outbound network connections from guests or services.", "type": "object", diff --git a/schema/crdb/blueprint-add-sled-state/up1.sql b/schema/crdb/blueprint-add-sled-state/up1.sql new file mode 100644 index 0000000000..855bc95ab4 --- /dev/null +++ b/schema/crdb/blueprint-add-sled-state/up1.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_state ( + blueprint_id UUID NOT NULL, + sled_id UUID NOT NULL, + sled_state omicron.public.sled_state NOT NULL, + PRIMARY KEY (blueprint_id, sled_id) +); diff --git a/schema/crdb/blueprint-add-sled-state/up2.sql b/schema/crdb/blueprint-add-sled-state/up2.sql new file mode 100644 index 0000000000..238870021f --- /dev/null +++ b/schema/crdb/blueprint-add-sled-state/up2.sql @@ -0,0 +1,14 @@ +set local disallow_full_table_scans = off; + +-- At this point in history, all sleds are considered active and this table only +-- exists to support transitioning active-but-expunged sleds to +-- 'decommissioned'. We'll fill in this table for all historical blueprints by +-- inserting rows for every sled for which a given blueprint had a zone config +-- with the state set to 'active'. +INSERT INTO bp_sled_state ( + SELECT DISTINCT + blueprint_id, + sled_id, + 'active'::sled_state + FROM bp_sled_omicron_zones +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index e7025f2499..e66f28d74f 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3262,6 +3262,16 @@ CREATE TABLE IF NOT EXISTS omicron.public.bp_target ( time_made_target TIMESTAMPTZ NOT NULL ); +-- state of a sled in a blueprint +CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_state ( + -- foreign key into `blueprint` table + blueprint_id UUID NOT NULL, + + sled_id UUID NOT NULL, + sled_state omicron.public.sled_state NOT NULL, + PRIMARY KEY (blueprint_id, sled_id) +); + -- description of a collection of omicron physical disks stored in a blueprint. CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_omicron_physical_disks ( -- foreign key into `blueprint` table @@ -3846,7 +3856,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '60.0.0', NULL) + (TRUE, NOW(), NOW(), '61.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 529fe833ff..27435686e7 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -95,6 +95,7 @@ use nexus_types::deployment::{ Blueprint, BlueprintPhysicalDisksConfig, BlueprintZoneConfig, BlueprintZoneDisposition, BlueprintZonesConfig, InvalidOmicronZoneType, }; +use nexus_types::external_api::views::SledState; use omicron_common::address::get_sled_address; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::ExternalPortDiscovery; @@ -1398,6 +1399,7 @@ pub(crate) fn build_initial_blueprint_from_sled_configs( } let mut blueprint_zones = BTreeMap::new(); + let mut sled_state = BTreeMap::new(); for (sled_id, sled_config) in sled_configs_by_id { let zones_config = BlueprintZonesConfig { // This is a bit of a hack. We only construct a blueprint after @@ -1419,12 +1421,14 @@ pub(crate) fn build_initial_blueprint_from_sled_configs( }; blueprint_zones.insert(*sled_id, zones_config); + sled_state.insert(*sled_id, SledState::Active); } Ok(Blueprint { id: Uuid::new_v4(), blueprint_zones, blueprint_disks, + sled_state, parent_blueprint_id: None, internal_dns_version, // We don't configure external DNS during RSS, so set it to an initial