pass blueprint during RSS <-> Nexus handoff

nexus/src/lib.rs

@@ -27,6 +27,7 @@ use dropshot::ConfigDropshot;
 use external_api::http_entrypoints::external_api;
 use internal_api::http_entrypoints::internal_api;
 use nexus_config::NexusConfig;
+use nexus_types::deployment::Blueprint;
 use nexus_types::external_api::views::SledProvisionPolicy;
 use nexus_types::internal_api::params::ServiceKind;
 use omicron_common::address::IpRange;
@@ -232,6 +233,7 @@ impl nexus_test_interface::NexusServer for Server {
     async fn start(
         internal_server: InternalServer,
         config: &NexusConfig,
+        blueprint: Blueprint,
         services: Vec<nexus_types::internal_api::params::ServicePutRequest>,
         datasets: Vec<nexus_types::internal_api::params::DatasetCreateRequest>,
         internal_dns_zone_config: nexus_types::internal_api::params::DnsConfigParams,
@@ -276,6 +278,7 @@ impl nexus_test_interface::NexusServer for Server {
                 &opctx,
                 config.deployment.rack_id,
                 internal_api::params::RackInitializationRequest {
+                    blueprint,
                     services,
                     datasets,
                     internal_services_ip_pool_ranges,

nexus/test-interface/src/lib.rs

@@ -33,6 +33,7 @@
 
 use async_trait::async_trait;
 use nexus_config::NexusConfig;
+use nexus_types::deployment::Blueprint;
 use slog::Logger;
 use std::net::{SocketAddr, SocketAddrV6};
 use uuid::Uuid;
@@ -50,6 +51,7 @@ pub trait NexusServer: Send + Sync + 'static {
     async fn start(
         internal_server: Self::InternalServer,
         config: &NexusConfig,
+        blueprint: Blueprint,
         services: Vec<nexus_types::internal_api::params::ServicePutRequest>,
         datasets: Vec<nexus_types::internal_api::params::DatasetCreateRequest>,
         internal_dns_config: nexus_types::internal_api::params::DnsConfigParams,

nexus/test-utils/src/lib.rs

@@ -7,6 +7,7 @@
 use anyhow::Context;
 use anyhow::Result;
 use camino::Utf8Path;
+use chrono::Utc;
 use dns_service_client::types::DnsConfigParams;
 use dropshot::test_util::ClientTestContext;
 use dropshot::test_util::LogContext;
@@ -24,6 +25,7 @@ use nexus_config::MgdConfig;
 use nexus_config::NexusConfig;
 use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
 use nexus_test_interface::NexusServer;
+use nexus_types::deployment::Blueprint;
 use nexus_types::external_api::params::UserId;
 use nexus_types::internal_api::params::Certificate;
 use nexus_types::internal_api::params::DatasetCreateRequest;
@@ -54,6 +56,8 @@ use oximeter_collector::Oximeter;
 use oximeter_producer::LogConfig;
 use oximeter_producer::Server as ProducerServer;
 use slog::{debug, error, o, Logger};
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
 use std::collections::HashMap;
 use std::fmt::Debug;
 use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6};
@@ -790,7 +794,17 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> {
             self.nexus_internal
                 .take()
                 .expect("Must launch internal nexus first"),
-            &self.config,
+            self.config,
+            Blueprint {
+                id: Uuid::new_v4(),
+                omicron_zones: BTreeMap::new(),
+                zones_in_service: BTreeSet::new(),
+                parent_blueprint_id: None,
+                internal_dns_version: Generation::new(),
+                time_created: Utc::now(),
+                creator: "fixme".to_string(),
+                comment: "fixme".to_string(),
+            },
             self.rack_init_builder.services.clone(),
             // NOTE: We should probably hand off
             // "self.rack_init_builder.datasets" here, but Nexus won't be happy

nexus/types/src/internal_api/params.rs

@@ -4,6 +4,7 @@
 
 //! Params define the request bodies of API endpoints for creating or updating resources.
 
+use crate::deployment::Blueprint;
 use crate::external_api::params::PhysicalDiskKind;
 use crate::external_api::params::UserId;
 use crate::external_api::shared::Baseboard;
@@ -248,6 +249,8 @@ impl std::fmt::Debug for Certificate {
 
 #[derive(Debug, Clone, Deserialize, JsonSchema)]
 pub struct RackInitializationRequest {
+    /// Blueprint describing services initialized by RSS.
+    pub blueprint: Blueprint,
     /// Services on the rack which have been created by RSS.
     pub services: Vec<ServicePutRequest>,
     /// Datasets on the rack which have been provisioned by RSS.

openapi/nexus-internal.json

@@ -5879,6 +5879,14 @@
       "RackInitializationRequest": {
         "type": "object",
         "properties": {
+          "blueprint": {
+            "description": "Blueprint describing services initialized by RSS.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Blueprint"
+              }
+            ]
+          },
           "certs": {
             "description": "x.509 Certificates used to encrypt communication with the external API.",
             "type": "array",
@@ -5945,6 +5953,7 @@
           }
         },
         "required": [
+          "blueprint",
           "certs",
           "datasets",
           "external_dns_zone_name",

sled-agent/src/rack_setup/mod.rs

@@ -9,3 +9,5 @@ pub mod config;
 mod plan;
 /// The main implementation of the RSS service.
 pub mod service;
+
+pub use plan::service::SledConfig;

sled-agent/src/rack_setup/service.rs

@@ -65,6 +65,7 @@
 //! thereafter.
 
 use super::config::SetupServiceConfig as Config;
+use super::plan::service::SledConfig;
 use crate::bootstrap::config::BOOTSTRAP_AGENT_HTTP_PORT;
 use crate::bootstrap::early_networking::{
     EarlyNetworkConfig, EarlyNetworkConfigBody, EarlyNetworkSetup,
@@ -107,7 +108,7 @@ use sled_hardware::underlay::BootstrapInterface;
 use sled_storage::dataset::CONFIG_DATASET;
 use sled_storage::manager::StorageHandle;
 use slog::Logger;
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::{btree_map, BTreeMap, BTreeSet};
 use std::collections::{HashMap, HashSet};
 use std::iter;
 use std::net::{Ipv6Addr, SocketAddrV6};
@@ -554,7 +555,7 @@ impl ServiceInner {
 
         // Build a Blueprint describing our service plan. This should never
         // fail, unless we've set up an invalid plan.
-        let _blueprint =
+        let blueprint =
             build_initial_blueprint_from_plan(sled_plan, service_plan)
                 .map_err(SetupServiceError::ConvertPlanToBlueprint)?;
 
@@ -665,6 +666,7 @@ impl ServiceInner {
         info!(self.log, "rack_network_config: {:#?}", rack_network_config);
 
         let request = NexusTypes::RackInitializationRequest {
+            blueprint,
             services,
             datasets,
             internal_services_ip_pool_ranges,
@@ -1105,20 +1107,38 @@ fn build_initial_blueprint_from_plan(
         Generation::try_from(service_plan.dns_config.generation)
             .context("invalid internal dns version")?;
 
-    let mut sled_id_by_addr = BTreeMap::new();
+    let mut sled_configs = BTreeMap::new();
     for sled_request in sled_plan.sleds.values() {
-        let addr = get_sled_address(sled_request.body.subnet);
-        if sled_id_by_addr.insert(addr, sled_request.body.id).is_some() {
-            bail!("duplicate sled address deriving blueprint: {addr}");
-        }
+        let sled_addr = get_sled_address(sled_request.body.subnet);
+        let sled_id = sled_request.body.id;
+        let entry = match sled_configs.entry(sled_id) {
+            btree_map::Entry::Vacant(entry) => entry,
+            btree_map::Entry::Occupied(_) => {
+                bail!("duplicate sled address deriving blueprint: {sled_addr}");
+            }
+        };
+        let sled_config =
+            service_plan.services.get(&sled_addr).with_context(|| {
+                format!(
+                    "missing services in plan for sled {sled_id} ({sled_addr})"
+                )
+            })?;
+        entry.insert(sled_config.clone());
     }
 
+    Ok(build_initial_blueprint_from_sled_configs(
+        sled_configs,
+        internal_dns_version,
+    ))
+}
+
+pub(crate) fn build_initial_blueprint_from_sled_configs(
+    sled_configs: BTreeMap<Uuid, SledConfig>,
+    internal_dns_version: Generation,
+) -> Blueprint {
     let mut omicron_zones = BTreeMap::new();
     let mut zones_in_service = BTreeSet::new();
-    for (sled_addr, sled_config) in &service_plan.services {
-        let sled_id = *sled_id_by_addr.get(&sled_addr).with_context(|| {
-            format!("could not sled ID for sled with address {sled_addr}")
-        })?;
+    for (sled_id, sled_config) in sled_configs {
         for zone in &sled_config.zones {
             zones_in_service.insert(zone.id);
         }
@@ -1135,18 +1155,13 @@ fn build_initial_blueprint_from_plan(
                 // value, we will need to revisit storing this in the serialized
                 // RSS plan.
                 generation: DeployStepVersion::V5_EVERYTHING,
-                zones: sled_config.zones.clone(),
+                zones: sled_config.zones,
             },
         );
-        if omicron_zones.insert(sled_id, zones_config).is_some() {
-            bail!(
-                "duplicate sled ID deriving blueprint: \
-                 {sled_id} (address={sled_addr}"
-            );
-        }
+        omicron_zones.insert(sled_id, zones_config);
     }
 
-    Ok(Blueprint {
+    Blueprint {
         id: Uuid::new_v4(),
         omicron_zones,
         zones_in_service,
@@ -1155,7 +1170,7 @@ fn build_initial_blueprint_from_plan(
         time_created: Utc::now(),
         creator: "RSS".to_string(),
         comment: "initial blueprint from rack setup".to_string(),
-    })
+    }
 }
 
 /// Facilitates creating a sequence of OmicronZonesConfig objects for each sled