diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index d475ec63d3..43588c0a85 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -5,7 +5,7 @@ //! Types shared between Nexus and Sled Agent. use crate::api::external::{self, Name}; -use ipnetwork::{IpNetwork, Ipv4Network}; +use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{ @@ -75,6 +75,7 @@ pub type RackNetworkConfig = RackNetworkConfigV1; /// Initial network configuration #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct RackNetworkConfigV1 { + pub rack_subnet: Ipv6Network, // TODO: #3591 Consider making infra-ip ranges implicit for uplinks /// First ip address to be used for configuring network infrastructure pub infra_ip_first: Ipv4Addr, diff --git a/nexus/db-model/src/rack.rs b/nexus/db-model/src/rack.rs index 0f1ef2a853..580ec155b4 100644 --- a/nexus/db-model/src/rack.rs +++ b/nexus/db-model/src/rack.rs @@ -4,6 +4,7 @@ use crate::schema::rack; use db_macros::Asset; +use ipnetwork::IpNetwork; use nexus_types::{external_api::views, identity::Asset}; use uuid::Uuid; @@ -15,6 +16,7 @@ pub struct Rack { pub identity: RackIdentity, pub initialized: bool, pub tuf_base_url: Option, + pub rack_subnet: Option, } impl Rack { @@ -23,6 +25,7 @@ impl Rack { identity: RackIdentity::new(id), initialized: false, tuf_base_url: None, + rack_subnet: None, } } } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index a4a7487f91..3aa3e6ff20 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -680,6 +680,7 @@ table! { time_modified -> Timestamptz, initialized -> Bool, tuf_base_url -> Nullable, + rack_subnet -> Nullable, } } diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index f5f7524aab..cb0daa557c 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -32,6 +32,7 @@ use chrono::Utc; use diesel::prelude::*; use diesel::result::Error as DieselError; use diesel::upsert::excluded; +use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::DnsZone; use nexus_db_model::ExternalIp; @@ -61,6 +62,7 @@ use uuid::Uuid; #[derive(Clone)] pub struct RackInit { pub rack_id: Uuid, + pub rack_subnet: IpNetwork, pub services: Vec, pub datasets: Vec, pub service_ip_pool_ranges: Vec, @@ -681,6 +683,7 @@ mod test { fn default() -> Self { RackInit { rack_id: Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap(), + rack_subnet: nexus_test_utils::RACK_SUBNET.parse().unwrap(), services: vec![], datasets: vec![], service_ip_pool_ranges: vec![], diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 3faae7f065..16535416fb 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -5,11 +5,15 @@ //! Rack management use super::silo::silo_dns_name; +use crate::external_api::params; use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; +use internal_dns::ServiceName; +use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -37,11 +41,17 @@ use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::Name; use omicron_common::api::external::NameOrId; +use omicron_common::api::external::SwitchLocation; use omicron_common::api::internal::shared::ExternalPortDiscovery; +use sled_agent_client::types::{ + BgpConfig, BgpPeerConfig, EarlyNetworkConfig, PortConfigV1, + RackNetworkConfig, RouteConfig as SledRouteConfig, +}; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; +use std::net::Ipv4Addr; use std::str::FromStr; use uuid::Uuid; @@ -186,10 +196,18 @@ impl super::Nexus { mapped_fleet_roles, }; + let rack_network_config = request.rack_network_config.as_ref().ok_or( + Error::InvalidRequest { + message: "cannot initialize a rack without a network config" + .into(), + }, + )?; + self.db_datastore .rack_set_initialized( opctx, RackInit { + rack_subnet: rack_network_config.rack_subnet.into(), rack_id, services: request.services, datasets, @@ -512,6 +530,7 @@ impl super::Nexus { } // TODO - https://github.com/oxidecomputer/omicron/issues/3277 // record port speed }; + self.initial_bootstore_sync(&opctx).await?; Ok(()) } @@ -545,4 +564,150 @@ impl super::Nexus { tokio::time::sleep(std::time::Duration::from_secs(2)).await; } } + + pub(crate) async fn initial_bootstore_sync( + &self, + opctx: &OpContext, + ) -> Result<(), Error> { + let mut rack = self.rack_lookup(opctx, &self.rack_id).await?; + if rack.rack_subnet.is_some() { + return Ok(()); + } + let addr = self + .resolver() + .await + .lookup_socket_v6(ServiceName::Scrimlet(SwitchLocation::Switch0)) + .await + .map_err(|e| Error::InternalError { + internal_message: e.to_string(), + })?; + + let sa = sled_agent_client::Client::new( + &format!("http://{}", addr), + self.log.clone(), + ); + + let result = sa + .read_network_bootstore_config() + .await + .map_err(|e| Error::InternalError { + internal_message: format!("read bootstore network config: {e}"), + })? + .into_inner(); + + rack.rack_subnet = + result.rack_network_config.map(|x| x.rack_subnet.into()); + + self.datastore().rack_insert(opctx, &rack).await?; + + Ok(()) + } + + pub(crate) async fn bootstore_network_config( + &self, + opctx: &OpContext, + ) -> Result { + let rack = self.rack_lookup(opctx, &self.rack_id).await?; + + let subnet = match rack.rack_subnet { + Some(IpNetwork::V6(subnet)) => subnet, + Some(IpNetwork::V4(_)) => { + return Err(Error::InternalError { + internal_message: "rack subnet not IPv6".into(), + }) + } + None => { + return Err(Error::InternalError { + internal_message: "rack subnet not set".into(), + }) + } + }; + + let db_ports = self.active_port_settings(opctx).await?; + let mut ports = Vec::new(); + let mut bgp = Vec::new(); + for (port, info) in &db_ports { + let mut peer_info = Vec::new(); + for p in &info.bgp_peers { + let bgp_config = + self.bgp_config_get(&opctx, p.bgp_config_id.into()).await?; + let announcements = self + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: bgp_config.bgp_announce_set_id.into(), + }, + ) + .await?; + let addr = match p.addr { + ipnetwork::IpNetwork::V4(addr) => addr, + ipnetwork::IpNetwork::V6(_) => continue, //TODO v6 + }; + peer_info.push((p, bgp_config.asn.0, addr.ip())); + bgp.push(BgpConfig { + asn: bgp_config.asn.0, + originate: announcements + .iter() + .filter_map(|a| match a.network { + IpNetwork::V4(net) => Some(net.into()), + //TODO v6 + _ => None, + }) + .collect(), + }); + } + + let p = PortConfigV1 { + routes: info + .routes + .iter() + .map(|r| SledRouteConfig { + destination: r.dst, + nexthop: r.gw.ip(), + }) + .collect(), + addresses: info.addresses.iter().map(|a| a.address).collect(), + bgp_peers: peer_info + .iter() + .map(|(_p, asn, addr)| BgpPeerConfig { + addr: *addr, + asn: *asn, + port: port.port_name.clone(), + }) + .collect(), + switch: port.switch_location.parse().unwrap(), + port: port.port_name.clone(), + uplink_port_fec: info + .links + .get(0) //TODO breakout support + .map(|l| l.fec) + .unwrap_or(SwitchLinkFec::None) + .into(), + uplink_port_speed: info + .links + .get(0) //TODO breakout support + .map(|l| l.speed) + .unwrap_or(SwitchLinkSpeed::Speed100G) + .into(), + }; + + ports.push(p); + } + + let result = EarlyNetworkConfig { + generation: 0, + ntp_servers: Vec::new(), //TODO + rack_network_config: Some(RackNetworkConfig { + rack_subnet: subnet, + //TODO(ry) you are here. We need to remove these too. They are + // inconsistent with a generic set of addresses on ports. + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports, + bgp, + }), + }; + + Ok(result) + } } diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index e2a9ef6e79..543ba77a51 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -559,22 +559,8 @@ async fn spa_undo_ensure_switch_port_bootstore_network_settings( // Just choosing the sled agent associated with switch0 for no reason. let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; - // Read the current bootstore network config. - let bs_config = read_bootstore_config(&sa).await?; - - // Compute the total network config from the nexus database. - let mut nexus_config = nexus - .compute_bootstore_network_config(&opctx, &bs_config) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "read nexus bootstore network config: {e}" - )) - })?; - - // Set the correct generation number and send the update. - nexus_config.generation = bs_config.generation; - write_bootstore_config(&sa, &nexus_config).await?; + let config = nexus.bootstore_network_config(&opctx).await?; + write_bootstore_config(&sa, &config).await?; Ok(()) } diff --git a/nexus/src/app/sagas/switch_port_settings_clear.rs b/nexus/src/app/sagas/switch_port_settings_clear.rs index 2836d5a2e2..259ca8cf1c 100644 --- a/nexus/src/app/sagas/switch_port_settings_clear.rs +++ b/nexus/src/app/sagas/switch_port_settings_clear.rs @@ -315,22 +315,12 @@ async fn spa_clear_switch_port_bootstore_network_settings( // Just choosing the sled agent associated with switch0 for no reason. let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; - // Read the current bootstore network config. - let bs_config = read_bootstore_config(&sa).await?; - - // Compute the total network config from the nexus database. - let mut nexus_config = nexus - .compute_bootstore_network_config(&opctx, &bs_config) - .await - .map_err(|e| { - ActionError::action_failed(format!( - "read nexus bootstore network config: {e}" - )) - })?; - - // Set the correct generation number and send the update. - nexus_config.generation = bs_config.generation; - write_bootstore_config(&sa, &nexus_config).await?; + let config = nexus.bootstore_network_config(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "read nexus bootstore network config: {e}" + )) + })?; + write_bootstore_config(&sa, &config).await?; Ok(()) } diff --git a/nexus/src/app/switch_port.rs b/nexus/src/app/switch_port.rs index 74dcd90e51..4d67389ad8 100644 --- a/nexus/src/app/switch_port.rs +++ b/nexus/src/app/switch_port.rs @@ -2,6 +2,9 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//XXX +#![allow(unused_imports)] + use crate::app::sagas; use crate::external_api::params; use db::datastore::SwitchPortSettingsCombinedResult; @@ -316,109 +319,4 @@ impl super::Nexus { LookupResult::Ok(ports) } - - pub(crate) async fn compute_bootstore_network_config( - &self, - opctx: &OpContext, - current: &EarlyNetworkConfig, - ) -> LookupResult { - let mut rack_net_config = match ¤t.rack_network_config { - Some(cfg) => { - RackNetworkConfig { - infra_ip_first: cfg.infra_ip_first, - infra_ip_last: cfg.infra_ip_last, - ports: Vec::new(), // To be filled in from db - bgp: Vec::new(), // To be filled in from db - } - } - None => { - return LookupResult::Err( - external::Error::ServiceUnavailable { - internal_message: - "bootstore network config not initialized yet" - .to_string(), - }, - ); - } - }; - - let db_ports = self.active_port_settings(opctx).await?; - - for (port, info) in &db_ports { - let mut peer_info = Vec::new(); - for p in &info.bgp_peers { - let bgp_config = - self.bgp_config_get(&opctx, p.bgp_config_id.into()).await?; - let announcements = self - .bgp_announce_list( - &opctx, - ¶ms::BgpAnnounceSetSelector { - name_or_id: bgp_config.bgp_announce_set_id.into(), - }, - ) - .await?; - let addr = match p.addr { - ipnetwork::IpNetwork::V4(addr) => addr, - ipnetwork::IpNetwork::V6(_) => continue, //TODO v6 - }; - peer_info.push((p, bgp_config.asn.0, addr.ip())); - rack_net_config.bgp.push(BgpConfig { - asn: bgp_config.asn.0, - originate: announcements - .iter() - .filter_map(|a| match a.network { - IpNetwork::V4(net) => Some(net.into()), - //TODO v6 - _ => None, - }) - .collect(), - }); - } - - let p = PortConfigV1 { - routes: info - .routes - .iter() - .map(|r| RouteConfig { - destination: r.dst, - nexthop: r.gw.ip(), - }) - .collect(), - addresses: info.addresses.iter().map(|a| a.address).collect(), - bgp_peers: peer_info - .iter() - .map(|(_p, asn, addr)| BgpPeerConfig { - addr: *addr, - asn: *asn, - port: port.port_name.clone(), - }) - .collect(), - switch: port.switch_location.parse().unwrap(), - port: port.port_name.clone(), - uplink_port_fec: info - .links - .get(0) //TODO breakout support - .map(|l| l.fec) - .unwrap_or(SwitchLinkFec::None) - .into(), - uplink_port_speed: info - .links - .get(0) //TODO breakout support - .map(|l| l.speed) - .unwrap_or(SwitchLinkSpeed::Speed100G) - .into(), - }; - - rack_net_config.ports.push(p); - } - - let result = EarlyNetworkConfig { - generation: current.generation, - rack_subnet: current.rack_subnet, - ntp_servers: current.ntp_servers.clone(), //TODO update from db - rack_network_config: Some(rack_net_config), - }; - - LookupResult::Ok(result) - } } diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index 586c828683..f4e3e07950 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -31,12 +31,12 @@ use internal_api::http_entrypoints::internal_api; use nexus_types::internal_api::params::ServiceKind; use omicron_common::address::IpRange; use omicron_common::api::internal::shared::{ - ExternalPortDiscovery, SwitchLocation, + ExternalPortDiscovery, RackNetworkConfig, SwitchLocation, }; use omicron_common::FileKv; use slog::Logger; use std::collections::HashMap; -use std::net::{SocketAddr, SocketAddrV6}; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -252,7 +252,13 @@ impl nexus_test_interface::NexusServer for Server { vec!["qsfp0".parse().unwrap()], )]), ), - rack_network_config: None, + rack_network_config: Some(RackNetworkConfig { + rack_subnet: "fd00:1122:3344:01::/56".parse().unwrap(), + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports: Vec::new(), + bgp: Vec::new(), + }), }, ) .await diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 2875363111..701a6e8ba9 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -58,6 +58,7 @@ pub const RACK_UUID: &str = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"; pub const SWITCH_UUID: &str = "dae4e1f1-410e-4314-bff1-fec0504be07e"; pub const OXIMETER_UUID: &str = "39e6175b-4df2-4730-b11d-cbc1e60a2e78"; pub const PRODUCER_UUID: &str = "a6458b7d-87c3-4483-be96-854d814c20de"; +pub const RACK_SUBNET: &str = "fd00:1122:3344:01::/56"; /// The reported amount of hardware threads for an emulated sled agent. pub const TEST_HARDWARE_THREADS: u32 = 16; diff --git a/nexus/tests/integration_tests/address_lots.rs b/nexus/tests/integration_tests/address_lots.rs index b4659daa62..40c8865929 100644 --- a/nexus/tests/integration_tests/address_lots.rs +++ b/nexus/tests/integration_tests/address_lots.rs @@ -27,8 +27,8 @@ type ControlPlaneTestContext = async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { let client = &ctx.external_client; - // Verify there are no lots - let lots = NexusRequest::iter_collection_authn::( + // Verify there is only one system lot + let lots = NexusRequest::iter_collection_authn::( client, "/v1/system/networking/address-lot", "", @@ -37,7 +37,7 @@ async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { .await .expect("Failed to list address lots") .all_items; - assert_eq!(lots.len(), 0, "Expected no lots"); + assert_eq!(lots.len(), 1, "Expected one lot"); // Create a lot let params = AddressLotCreate { @@ -111,8 +111,8 @@ async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { .expect("Failed to list address lots") .all_items; - assert_eq!(lots.len(), 1, "Expected 1 lot"); - assert_eq!(lots[0], address_lot); + assert_eq!(lots.len(), 2, "Expected 2 lots"); + assert_eq!(lots[1], address_lot); // Verify there are lot blocks let blist = NexusRequest::iter_collection_authn::( diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 8bfdbeae2a..006e11da97 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -690,13 +690,17 @@ "items": { "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ "bgp", "infra_ip_first", "infra_ip_last", - "ports" + "ports", + "rack_subnet" ] }, "RackOperationStatus": { diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 1c1d29fd8b..47fb0fb056 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -4460,13 +4460,17 @@ "items": { "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ "bgp", "infra_ip_first", "infra_ip_last", - "ports" + "ports", + "rack_subnet" ] }, "RecoverySiloConfig": { diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index d22dff4362..05eac57d4f 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -1733,15 +1733,10 @@ "$ref": "#/components/schemas/RackNetworkConfigV1" } ] - }, - "rack_subnet": { - "type": "string", - "format": "ipv6" } }, "required": [ - "ntp_servers", - "rack_subnet" + "ntp_servers" ] }, "Error": { @@ -2610,13 +2605,17 @@ "items": { "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ "bgp", "infra_ip_first", "infra_ip_last", - "ports" + "ports", + "rack_subnet" ] }, "RouteConfig": { diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 1bd73d3fd4..5a5ee337ff 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -2133,13 +2133,17 @@ "items": { "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ "bgp", "infra_ip_first", "infra_ip_last", - "ports" + "ports", + "rack_subnet" ] }, "RackOperationStatus": { diff --git a/schema/crdb/8.0.0/up4.sql b/schema/crdb/8.0.0/up4.sql new file mode 100644 index 0000000000..44bfd90b8c --- /dev/null +++ b/schema/crdb/8.0.0/up4.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.rack ADD COLUMN IF NOT EXISTS rack_subnet INET; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 31894a36a2..df4dc78405 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -63,7 +63,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.rack ( initialized BOOL NOT NULL, /* Used to configure the updates service URL */ - tuf_base_url STRING(512) + tuf_base_url STRING(512), + + /* The IPv6 underlay /56 prefix for the rack */ + rack_subnet INET ); /* diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 2d35c2a355..e1ef503a6a 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -510,7 +510,8 @@ "bgp", "infra_ip_first", "infra_ip_last", - "ports" + "ports", + "rack_subnet" ], "properties": { "bgp": { @@ -536,6 +537,9 @@ "items": { "$ref": "#/definitions/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/definitions/Ipv6Network" } } }, diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 949e4d3082..21922a7540 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -16,8 +16,8 @@ use futures::future; use gateway_client::Client as MgsClient; use internal_dns::resolver::{ResolveError, Resolver as DnsResolver}; use internal_dns::ServiceName; -use ipnetwork::IpNetwork; -use omicron_common::address::{Ipv6Subnet, AZ_PREFIX, MGS_PORT}; +use ipnetwork::{IpNetwork, Ipv6Network}; +use omicron_common::address::{Ipv6Subnet, MGS_PORT}; use omicron_common::address::{DDMD_PORT, DENDRITE_PORT}; use omicron_common::api::internal::shared::{ PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, RackNetworkConfigV1, @@ -564,6 +564,31 @@ fn retry_policy_switch_mapping() -> ExponentialBackoff { .build() } +// The first production version of the `EarlyNetworkConfig`. +// +// If this version is in the bootstore than we need to convert it to +// `EarlyNetworkConfigV1`. +// +// Once we do this for all customers that have initialized racks with the +// old version we can go ahead and remove this type and its conversion code +// altogether. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +struct EarlyNetworkConfigV0 { + // The current generation number of data as stored in CRDB. + // The initial generation is set during RSS time and then only mutated + // by Nexus. + pub generation: u64, + + pub rack_subnet: Ipv6Addr, + + /// The external NTP server addresses. + pub ntp_servers: Vec, + + // Rack network configuration as delivered from RSS and only existing at + // generation 1 + pub rack_network_config: Option, +} + /// Network configuration required to bring up the control plane /// /// The fields in this structure are those from @@ -594,8 +619,6 @@ pub struct EarlyNetworkConfig { /// future (post-v1) deserialization paths. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct EarlyNetworkConfigBody { - pub rack_subnet: Ipv6Addr, - /// The external NTP server addresses. pub ntp_servers: Vec, @@ -603,37 +626,6 @@ pub struct EarlyNetworkConfigBody { pub rack_network_config: Option, } -// The first production version of the `EarlyNetworkConfig`. -// -// If this version is in the bootstore than we need to convert it to -// `EarlyNetworkConfigV1`. -// -// Once we do this for all customers that have initialized racks with the -// old version we can go ahead and remove this type and its conversion code -// altogether. -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -struct EarlyNetworkConfigV0 { - // The current generation number of data as stored in CRDB. - // The initial generation is set during RSS time and then only mutated - // by Nexus. - pub generation: u64, - - pub rack_subnet: Ipv6Addr, - - /// The external NTP server addresses. - pub ntp_servers: Vec, - - // Rack network configuration as delivered from RSS and only existing at - // generation 1 - pub rack_network_config: Option, -} - -impl EarlyNetworkConfig { - pub fn az_subnet(&self) -> Ipv6Subnet { - Ipv6Subnet::::new(self.body.rack_subnet) - } -} - impl From for bootstore::NetworkConfig { fn from(value: EarlyNetworkConfig) -> Self { // Can this ever actually fail? @@ -671,11 +663,10 @@ impl TryFrom for EarlyNetworkConfig { generation: v0.generation, schema_version: 1, body: EarlyNetworkConfigBody { - rack_subnet: v0.rack_subnet, ntp_servers: v0.ntp_servers, - rack_network_config: v0 - .rack_network_config - .map(|v0_config| RackNetworkConfigV1::from(v0_config)), + rack_network_config: v0.rack_network_config.map(|v0_config| { + RackNetworkConfigV0::to_v1(v0.rack_subnet, v0_config) + }), }, }) } @@ -697,12 +688,22 @@ pub struct RackNetworkConfigV0 { pub uplinks: Vec, } -impl From for RackNetworkConfigV1 { - fn from(value: RackNetworkConfigV0) -> Self { +impl RackNetworkConfigV0 { + /// Convert from `RackNetworkConfigV0` to `RackNetworkConfigV1` + /// + /// We cannot use `From for `RackNetworkConfigV1` + /// because the `rack_network` field does not exist in `RackNetworkConfigV0` + /// and must be passed in from the `EarlyNetworkConfigV0` struct which + /// contains the `RackNetworkConfivV0` struct. + pub fn to_v1( + rack_subnet: Ipv6Addr, + v0: RackNetworkConfigV0, + ) -> RackNetworkConfigV1 { RackNetworkConfigV1 { - infra_ip_first: value.infra_ip_first, - infra_ip_last: value.infra_ip_last, - ports: value + rack_subnet: Ipv6Network::new(rack_subnet, 56).unwrap(), + infra_ip_first: v0.infra_ip_first, + infra_ip_last: v0.infra_ip_last, + ports: v0 .uplinks .into_iter() .map(|uplink| PortConfigV1::from(uplink)) diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 7b15ad1171..eef05202be 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -577,6 +577,7 @@ impl ServiceInner { let rack_network_config = match &config.rack_network_config { Some(config) => { let value = NexusTypes::RackNetworkConfig { + rack_subnet: config.rack_subnet, infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, ports: config @@ -898,7 +899,6 @@ impl ServiceInner { generation: 1, schema_version: 1, body: EarlyNetworkConfigBody { - rack_subnet: config.rack_subnet, ntp_servers: config.ntp_servers.clone(), rack_network_config: config.rack_network_config.clone(), }, diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 797cb3535f..f77da11b0e 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -22,6 +22,7 @@ use dropshot::RequestContext; use dropshot::TypedBody; use illumos_utils::opte::params::DeleteVirtualNetworkInterfaceHost; use illumos_utils::opte::params::SetVirtualNetworkInterfaceHost; +use ipnetwork::Ipv6Network; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::UpdateArtifactId; @@ -359,9 +360,10 @@ async fn read_network_bootstore_config( generation: 0, schema_version: 1, body: EarlyNetworkConfigBody { - rack_subnet: Ipv6Addr::UNSPECIFIED, ntp_servers: Vec::new(), rack_network_config: Some(RackNetworkConfig { + rack_subnet: Ipv6Network::new(Ipv6Addr::UNSPECIFIED, 56) + .unwrap(), infra_ip_first: Ipv4Addr::UNSPECIFIED, infra_ip_last: Ipv4Addr::UNSPECIFIED, ports: Vec::new(), diff --git a/smf/sled-agent/gimlet-standalone/config-rss.toml b/smf/sled-agent/gimlet-standalone/config-rss.toml index 636e846b7a..29a7a79eba 100644 --- a/smf/sled-agent/gimlet-standalone/config-rss.toml +++ b/smf/sled-agent/gimlet-standalone/config-rss.toml @@ -88,6 +88,7 @@ last = "192.168.1.29" # Configuration to bring up Boundary Services and make Nexus reachable from the # outside. See docs/how-to-run.adoc for more on what to put here. [rack_network_config] +rack_subnet = "fd00:1122:3344:01::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index bbc42b3375..fea3cfa5d8 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -88,6 +88,7 @@ last = "192.168.1.29" # Configuration to bring up Boundary Services and make Nexus reachable from the # outside. See docs/how-to-run.adoc for more on what to put here. [rack_network_config] +rack_subnet = "fd00:1122:3344:01::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. diff --git a/wicket/src/rack_setup/config_template.toml b/wicket/src/rack_setup/config_template.toml index dd371c7628..216daacaa0 100644 --- a/wicket/src/rack_setup/config_template.toml +++ b/wicket/src/rack_setup/config_template.toml @@ -40,6 +40,7 @@ bootstrap_sleds = [] # TODO: docs on network config [rack_network_config] +rack_subnet = "" infra_ip_first = "" infra_ip_last = "" diff --git a/wicket/src/rack_setup/config_toml.rs b/wicket/src/rack_setup/config_toml.rs index 43af459324..0d5bc9c948 100644 --- a/wicket/src/rack_setup/config_toml.rs +++ b/wicket/src/rack_setup/config_toml.rs @@ -195,6 +195,7 @@ fn populate_network_table( }; for (property, value) in [ + ("rack_subnet", config.rack_subnet.to_string()), ("infra_ip_first", config.infra_ip_first.to_string()), ("infra_ip_last", config.infra_ip_last.to_string()), ] { @@ -365,6 +366,7 @@ mod tests { external_dns_ips: value.external_dns_ips, ntp_servers: value.ntp_servers, rack_network_config: InternalRackNetworkConfig { + rack_subnet: rnc.rack_subnet, infra_ip_first: rnc.infra_ip_first, infra_ip_last: rnc.infra_ip_last, ports: rnc @@ -471,6 +473,7 @@ mod tests { external_dns_ips: vec!["10.0.0.1".parse().unwrap()], ntp_servers: vec!["ntp1.com".into(), "ntp2.com".into()], rack_network_config: Some(RackNetworkConfig { + rack_subnet: "fd00:1122:3344:01::/56".parse().unwrap(), infra_ip_first: "172.30.0.1".parse().unwrap(), infra_ip_last: "172.30.0.10".parse().unwrap(), ports: vec![PortConfigV1 { diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index f335754318..35e9ef383e 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -497,6 +497,7 @@ fn validate_rack_network_config( // TODO Add more client side checks on `rack_network_config` contents? Ok(bootstrap_agent_client::types::RackNetworkConfig { + rack_subnet: config.rack_subnet, infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, ports: config