diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index c0fba31afb..03470a4f3a 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -27,6 +27,7 @@ use dropshot::ConfigDropshot; use external_api::http_entrypoints::external_api; use internal_api::http_entrypoints::internal_api; use nexus_config::NexusConfig; +use nexus_types::deployment::Blueprint; use nexus_types::external_api::views::SledProvisionPolicy; use nexus_types::internal_api::params::ServiceKind; use omicron_common::address::IpRange; @@ -232,6 +233,7 @@ impl nexus_test_interface::NexusServer for Server { async fn start( internal_server: InternalServer, config: &NexusConfig, + blueprint: Blueprint, services: Vec, datasets: Vec, internal_dns_zone_config: nexus_types::internal_api::params::DnsConfigParams, @@ -276,6 +278,7 @@ impl nexus_test_interface::NexusServer for Server { &opctx, config.deployment.rack_id, internal_api::params::RackInitializationRequest { + blueprint, services, datasets, internal_services_ip_pool_ranges, diff --git a/nexus/test-interface/src/lib.rs b/nexus/test-interface/src/lib.rs index 10bc9e63f0..d9bb69276a 100644 --- a/nexus/test-interface/src/lib.rs +++ b/nexus/test-interface/src/lib.rs @@ -33,6 +33,7 @@ use async_trait::async_trait; use nexus_config::NexusConfig; +use nexus_types::deployment::Blueprint; use slog::Logger; use std::net::{SocketAddr, SocketAddrV6}; use uuid::Uuid; @@ -50,6 +51,7 @@ pub trait NexusServer: Send + Sync + 'static { async fn start( internal_server: Self::InternalServer, config: &NexusConfig, + blueprint: Blueprint, services: Vec, datasets: Vec, internal_dns_config: nexus_types::internal_api::params::DnsConfigParams, diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 9681d9ff97..a557cef3dc 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -7,6 +7,7 @@ use anyhow::Context; use anyhow::Result; use camino::Utf8Path; +use chrono::Utc; use dns_service_client::types::DnsConfigParams; use dropshot::test_util::ClientTestContext; use dropshot::test_util::LogContext; @@ -24,6 +25,7 @@ use nexus_config::MgdConfig; use nexus_config::NexusConfig; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use nexus_test_interface::NexusServer; +use nexus_types::deployment::Blueprint; use nexus_types::external_api::params::UserId; use nexus_types::internal_api::params::Certificate; use nexus_types::internal_api::params::DatasetCreateRequest; @@ -54,6 +56,8 @@ use oximeter_collector::Oximeter; use oximeter_producer::LogConfig; use oximeter_producer::Server as ProducerServer; use slog::{debug, error, o, Logger}; +use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::collections::HashMap; use std::fmt::Debug; use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; @@ -790,7 +794,17 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { self.nexus_internal .take() .expect("Must launch internal nexus first"), - &self.config, + self.config, + Blueprint { + id: Uuid::new_v4(), + omicron_zones: BTreeMap::new(), + zones_in_service: BTreeSet::new(), + parent_blueprint_id: None, + internal_dns_version: Generation::new(), + time_created: Utc::now(), + creator: "fixme".to_string(), + comment: "fixme".to_string(), + }, self.rack_init_builder.services.clone(), // NOTE: We should probably hand off // "self.rack_init_builder.datasets" here, but Nexus won't be happy diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index e8c4703008..9f80d313fd 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -4,6 +4,7 @@ //! Params define the request bodies of API endpoints for creating or updating resources. +use crate::deployment::Blueprint; use crate::external_api::params::PhysicalDiskKind; use crate::external_api::params::UserId; use crate::external_api::shared::Baseboard; @@ -248,6 +249,8 @@ impl std::fmt::Debug for Certificate { #[derive(Debug, Clone, Deserialize, JsonSchema)] pub struct RackInitializationRequest { + /// Blueprint describing services initialized by RSS. + pub blueprint: Blueprint, /// Services on the rack which have been created by RSS. pub services: Vec, /// Datasets on the rack which have been provisioned by RSS. diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 9e18c7d6bc..ecc90fa646 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -5879,6 +5879,14 @@ "RackInitializationRequest": { "type": "object", "properties": { + "blueprint": { + "description": "Blueprint describing services initialized by RSS.", + "allOf": [ + { + "$ref": "#/components/schemas/Blueprint" + } + ] + }, "certs": { "description": "x.509 Certificates used to encrypt communication with the external API.", "type": "array", @@ -5945,6 +5953,7 @@ } }, "required": [ + "blueprint", "certs", "datasets", "external_dns_zone_name", diff --git a/sled-agent/src/rack_setup/mod.rs b/sled-agent/src/rack_setup/mod.rs index cfabca6209..0ad8e0ce71 100644 --- a/sled-agent/src/rack_setup/mod.rs +++ b/sled-agent/src/rack_setup/mod.rs @@ -9,3 +9,5 @@ pub mod config; mod plan; /// The main implementation of the RSS service. pub mod service; + +pub use plan::service::SledConfig; diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 7be85b5447..952240bd74 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -65,6 +65,7 @@ //! thereafter. use super::config::SetupServiceConfig as Config; +use super::plan::service::SledConfig; use crate::bootstrap::config::BOOTSTRAP_AGENT_HTTP_PORT; use crate::bootstrap::early_networking::{ EarlyNetworkConfig, EarlyNetworkConfigBody, EarlyNetworkSetup, @@ -107,7 +108,7 @@ use sled_hardware::underlay::BootstrapInterface; use sled_storage::dataset::CONFIG_DATASET; use sled_storage::manager::StorageHandle; use slog::Logger; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{btree_map, BTreeMap, BTreeSet}; use std::collections::{HashMap, HashSet}; use std::iter; use std::net::{Ipv6Addr, SocketAddrV6}; @@ -554,7 +555,7 @@ impl ServiceInner { // Build a Blueprint describing our service plan. This should never // fail, unless we've set up an invalid plan. - let _blueprint = + let blueprint = build_initial_blueprint_from_plan(sled_plan, service_plan) .map_err(SetupServiceError::ConvertPlanToBlueprint)?; @@ -665,6 +666,7 @@ impl ServiceInner { info!(self.log, "rack_network_config: {:#?}", rack_network_config); let request = NexusTypes::RackInitializationRequest { + blueprint, services, datasets, internal_services_ip_pool_ranges, @@ -1105,20 +1107,38 @@ fn build_initial_blueprint_from_plan( Generation::try_from(service_plan.dns_config.generation) .context("invalid internal dns version")?; - let mut sled_id_by_addr = BTreeMap::new(); + let mut sled_configs = BTreeMap::new(); for sled_request in sled_plan.sleds.values() { - let addr = get_sled_address(sled_request.body.subnet); - if sled_id_by_addr.insert(addr, sled_request.body.id).is_some() { - bail!("duplicate sled address deriving blueprint: {addr}"); - } + let sled_addr = get_sled_address(sled_request.body.subnet); + let sled_id = sled_request.body.id; + let entry = match sled_configs.entry(sled_id) { + btree_map::Entry::Vacant(entry) => entry, + btree_map::Entry::Occupied(_) => { + bail!("duplicate sled address deriving blueprint: {sled_addr}"); + } + }; + let sled_config = + service_plan.services.get(&sled_addr).with_context(|| { + format!( + "missing services in plan for sled {sled_id} ({sled_addr})" + ) + })?; + entry.insert(sled_config.clone()); } + Ok(build_initial_blueprint_from_sled_configs( + sled_configs, + internal_dns_version, + )) +} + +pub(crate) fn build_initial_blueprint_from_sled_configs( + sled_configs: BTreeMap, + internal_dns_version: Generation, +) -> Blueprint { let mut omicron_zones = BTreeMap::new(); let mut zones_in_service = BTreeSet::new(); - for (sled_addr, sled_config) in &service_plan.services { - let sled_id = *sled_id_by_addr.get(&sled_addr).with_context(|| { - format!("could not sled ID for sled with address {sled_addr}") - })?; + for (sled_id, sled_config) in sled_configs { for zone in &sled_config.zones { zones_in_service.insert(zone.id); } @@ -1135,18 +1155,13 @@ fn build_initial_blueprint_from_plan( // value, we will need to revisit storing this in the serialized // RSS plan. generation: DeployStepVersion::V5_EVERYTHING, - zones: sled_config.zones.clone(), + zones: sled_config.zones, }, ); - if omicron_zones.insert(sled_id, zones_config).is_some() { - bail!( - "duplicate sled ID deriving blueprint: \ - {sled_id} (address={sled_addr}" - ); - } + omicron_zones.insert(sled_id, zones_config); } - Ok(Blueprint { + Blueprint { id: Uuid::new_v4(), omicron_zones, zones_in_service, @@ -1155,7 +1170,7 @@ fn build_initial_blueprint_from_plan( time_created: Utc::now(), creator: "RSS".to_string(), comment: "initial blueprint from rack setup".to_string(), - }) + } } /// Facilitates creating a sequence of OmicronZonesConfig objects for each sled diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index dd815775ff..b7577e6d02 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -10,21 +10,30 @@ use super::sled_agent::SledAgent; use super::storage::PantryServer; use crate::nexus::d2n_params; use crate::nexus::NexusClient; +use crate::params::OmicronZoneConfig; +use crate::params::OmicronZoneDataset; +use crate::params::OmicronZoneType; +use crate::rack_setup::service::build_initial_blueprint_from_sled_configs; +use crate::rack_setup::SledConfig; use anyhow::anyhow; use crucible_agent_client::types::State as RegionState; +use illumos_utils::zpool::ZpoolName; use internal_dns::ServiceName; use nexus_client::types as NexusTypes; use nexus_client::types::{IpRange, Ipv4Range, Ipv6Range}; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; +use nexus_types::inventory::NetworkInterfaceKind; use omicron_common::address::DNS_OPTE_IPV4_SUBNET; use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; use omicron_common::api::external::Generation; use omicron_common::api::external::MacAddr; +use omicron_common::api::external::Vni; use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, BackoffError, }; use omicron_common::FileKv; use slog::{info, Drain, Logger}; +use std::collections::BTreeMap; use std::collections::HashMap; use std::net::IpAddr; use std::net::Ipv4Addr; @@ -334,6 +343,8 @@ pub async fn run_standalone_server( // Initialize the internal DNS entries let dns_config = dns_config_builder.build(); dns.initialize_with_config(&log, &dns_config).await?; + let internal_dns_version = Generation::try_from(dns_config.generation) + .expect("invalid internal dns version"); // Record the internal DNS server as though RSS had provisioned it so // that Nexus knows about it. @@ -341,37 +352,58 @@ pub async fn run_standalone_server( SocketAddr::V4(_) => panic!("did not expect v4 address"), SocketAddr::V6(a) => a, }; - let mut services = vec![NexusTypes::ServicePutRequest { - address: http_bound.to_string(), - kind: NexusTypes::ServiceKind::InternalDns, - service_id: Uuid::new_v4(), - sled_id: config.id, - zone_id: Some(Uuid::new_v4()), + let mut zones = vec![OmicronZoneConfig { + id: Uuid::new_v4(), + underlay_address: *http_bound.ip(), + zone_type: OmicronZoneType::InternalDns { + dataset: OmicronZoneDataset { + pool_name: ZpoolName::new_external(Uuid::new_v4()), + }, + http_address: http_bound, + dns_address: match dns.dns_server.local_address() { + SocketAddr::V4(_) => panic!("did not expect v4 address"), + SocketAddr::V6(a) => a, + }, + gz_address: Ipv6Addr::LOCALHOST, + gz_address_index: 0, + }, }]; let mut internal_services_ip_pool_ranges = vec![]; let mut macs = MacAddr::iter_system(); if let Some(nexus_external_addr) = rss_args.nexus_external_addr { let ip = nexus_external_addr.ip(); + let id = Uuid::new_v4(); - services.push(NexusTypes::ServicePutRequest { - address: config.nexus_address.to_string(), - kind: NexusTypes::ServiceKind::Nexus { - external_address: ip, - nic: NexusTypes::ServiceNic { + zones.push(OmicronZoneConfig { + id, + underlay_address: match ip { + IpAddr::V4(_) => panic!("did not expect v4 address"), + IpAddr::V6(a) => a, + }, + zone_type: OmicronZoneType::Nexus { + internal_address: match config.nexus_address { + SocketAddr::V4(_) => panic!("did not expect v4 address"), + SocketAddr::V6(a) => a, + }, + external_ip: ip, + nic: nexus_types::inventory::NetworkInterface { id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Service { id }, name: "nexus".parse().unwrap(), ip: NEXUS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES as u32 + 1) .unwrap() .into(), mac: macs.next().unwrap(), + subnet: (*NEXUS_OPTE_IPV4_SUBNET).into(), + vni: Vni::SERVICES_VNI, + primary: true, slot: 0, }, + external_tls: false, + external_dns_servers: vec![], }, - service_id: Uuid::new_v4(), - sled_id: config.id, - zone_id: Some(Uuid::new_v4()), }); internal_services_ip_pool_ranges.push(match ip { @@ -388,24 +420,31 @@ pub async fn run_standalone_server( rss_args.external_dns_internal_addr { let ip = *external_dns_internal_addr.ip(); - services.push(NexusTypes::ServicePutRequest { - address: external_dns_internal_addr.to_string(), - kind: NexusTypes::ServiceKind::ExternalDns { - external_address: ip.into(), - nic: NexusTypes::ServiceNic { + let id = Uuid::new_v4(); + zones.push(OmicronZoneConfig { + id, + underlay_address: ip, + zone_type: OmicronZoneType::ExternalDns { + dataset: OmicronZoneDataset { + pool_name: ZpoolName::new_external(Uuid::new_v4()), + }, + http_address: external_dns_internal_addr, + dns_address: SocketAddr::V6(external_dns_internal_addr), + nic: nexus_types::inventory::NetworkInterface { id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Service { id }, name: "external-dns".parse().unwrap(), ip: DNS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES as u32 + 1) .unwrap() .into(), mac: macs.next().unwrap(), + subnet: (*DNS_OPTE_IPV4_SUBNET).into(), + vni: Vni::SERVICES_VNI, + primary: true, slot: 0, }, }, - service_id: Uuid::new_v4(), - sled_id: config.id, - zone_id: Some(Uuid::new_v4()), }); internal_services_ip_pool_ranges @@ -450,7 +489,16 @@ pub async fn run_standalone_server( None => vec![], }; + let services = + zones.iter().map(|z| z.to_nexus_service_req(config.id)).collect(); + let mut sled_configs = BTreeMap::new(); + sled_configs.insert(config.id, SledConfig { zones }); + let rack_init_request = NexusTypes::RackInitializationRequest { + blueprint: build_initial_blueprint_from_sled_configs( + sled_configs, + internal_dns_version, + ), services, datasets, internal_services_ip_pool_ranges,