From 8f92aceba97e8d87e6bf742059a53295d0843193 Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Thu, 1 Jun 2023 23:45:13 +0000 Subject: [PATCH 1/3] Perform rack network initialization earlier in rack setup --- Cargo.lock | 1 + common/Cargo.toml | 1 + common/src/api/internal/shared.rs | 42 +++++++ ddm-admin-client/src/lib.rs | 5 +- docs/how-to-run.adoc | 1 + nexus-client/src/lib.rs | 36 ++++++ nexus/src/app/rack.rs | 86 +------------- openapi/bootstrap-agent.json | 26 ++++- openapi/nexus-internal.json | 26 ++++- sled-agent/src/rack_setup/service.rs | 131 ++++++++++++++++++++++ smf/sled-agent/non-gimlet/config-rss.toml | 1 + 11 files changed, 270 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7538028cb..5e04362cdd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4367,6 +4367,7 @@ dependencies = [ "api_identity 0.1.0", "backoff", "chrono", + "dpd-client", "dropshot", "expectorate", "futures", diff --git a/common/Cargo.toml b/common/Cargo.toml index 2aba145e90..0d8366f5a1 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -9,6 +9,7 @@ anyhow.workspace = true api_identity.workspace = true backoff.workspace = true chrono.workspace = true +dpd-client.workspace = true dropshot.workspace = true futures.workspace = true hex.workspace = true diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 181604a17d..680be80ed3 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -73,6 +73,48 @@ pub struct RackNetworkConfig { pub infra_ip_last: String, /// Switchport to use for external connectivity pub uplink_port: String, + /// Speed for the Switchport + pub uplink_port_speed: PortSpeed, /// IP Address to apply to switchport (must be in infra_ip pool) pub uplink_ip: String, } + +/// Switchport Speed options +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PortSpeed { + #[serde(alias = "0G")] + Speed0G, + #[serde(alias = "1G")] + Speed1G, + #[serde(alias = "10G")] + Speed10G, + #[serde(alias = "25G")] + Speed25G, + #[serde(alias = "40G")] + Speed40G, + #[serde(alias = "50G")] + Speed50G, + #[serde(alias = "100G")] + Speed100G, + #[serde(alias = "200G")] + Speed200G, + #[serde(alias = "400G")] + Speed400G, +} + +impl From for dpd_client::types::PortSpeed { + fn from(value: PortSpeed) -> Self { + match value { + PortSpeed::Speed0G => dpd_client::types::PortSpeed::Speed0G, + PortSpeed::Speed1G => dpd_client::types::PortSpeed::Speed1G, + PortSpeed::Speed10G => dpd_client::types::PortSpeed::Speed10G, + PortSpeed::Speed25G => dpd_client::types::PortSpeed::Speed25G, + PortSpeed::Speed40G => dpd_client::types::PortSpeed::Speed40G, + PortSpeed::Speed50G => dpd_client::types::PortSpeed::Speed50G, + PortSpeed::Speed100G => dpd_client::types::PortSpeed::Speed100G, + PortSpeed::Speed200G => dpd_client::types::PortSpeed::Speed200G, + PortSpeed::Speed400G => dpd_client::types::PortSpeed::Speed400G, + } + } +} diff --git a/ddm-admin-client/src/lib.rs b/ddm-admin-client/src/lib.rs index 997a41ba76..93248c73a1 100644 --- a/ddm-admin-client/src/lib.rs +++ b/ddm-admin-client/src/lib.rs @@ -60,7 +60,10 @@ impl Client { Self::new(log, SocketAddrV6::new(Ipv6Addr::LOCALHOST, DDMD_PORT, 0, 0)) } - fn new(log: &Logger, ddmd_addr: SocketAddrV6) -> Result { + pub fn new( + log: &Logger, + ddmd_addr: SocketAddrV6, + ) -> Result { let dur = std::time::Duration::from_secs(60); let log = log.new(slog::o!("DdmAdminClient" => SocketAddr::V6(ddmd_addr))); diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 59f39078a6..1b27255ddc 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -166,6 +166,7 @@ gateway_ip = "192.168.1.199" <--- ip address of your default gateway / router infra_ip_first = "192.168.1.30" <--- first address of pool used for rack networking infra_ip_last = "192.168.1.50" <--- last address of pool used for rack networking uplink_port = "qsfp0" <--- for "softnpu" environments, this will always be "qsfp0" +uplink_port_speed = "40G" <--- uplink interface speed uplink_ip = "192.168.1.32" <--- address to assign to the uplink port ---- diff --git a/nexus-client/src/lib.rs b/nexus-client/src/lib.rs index d634114d91..234284acbc 100644 --- a/nexus-client/src/lib.rs +++ b/nexus-client/src/lib.rs @@ -272,3 +272,39 @@ impl From<&omicron_common::api::internal::shared::SourceNatConfig> Self { ip: r.ip, first_port: r.first_port, last_port: r.last_port } } } + +impl From + for types::PortSpeed +{ + fn from(value: omicron_common::api::internal::shared::PortSpeed) -> Self { + match value { + omicron_common::api::internal::shared::PortSpeed::Speed0G => { + types::PortSpeed::Speed0G + } + omicron_common::api::internal::shared::PortSpeed::Speed1G => { + types::PortSpeed::Speed1G + } + omicron_common::api::internal::shared::PortSpeed::Speed10G => { + types::PortSpeed::Speed10G + } + omicron_common::api::internal::shared::PortSpeed::Speed25G => { + types::PortSpeed::Speed25G + } + omicron_common::api::internal::shared::PortSpeed::Speed40G => { + types::PortSpeed::Speed40G + } + omicron_common::api::internal::shared::PortSpeed::Speed50G => { + types::PortSpeed::Speed50G + } + omicron_common::api::internal::shared::PortSpeed::Speed100G => { + types::PortSpeed::Speed100G + } + omicron_common::api::internal::shared::PortSpeed::Speed200G => { + types::PortSpeed::Speed200G + } + omicron_common::api::internal::shared::PortSpeed::Speed400G => { + types::PortSpeed::Speed400G + } + } + } +} diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 8978bdc822..8fa361a937 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -11,16 +11,6 @@ use crate::db::lookup::LookupPath; use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; -use dpd_client::types::Ipv6Entry; -use dpd_client::types::LinkCreate; -use dpd_client::types::LinkId; -use dpd_client::types::LinkSettings; -use dpd_client::types::PortFec; -use dpd_client::types::PortId; -use dpd_client::types::PortSettings; -use dpd_client::types::PortSpeed; -use dpd_client::types::RouteSettingsV4; -use dpd_client::Ipv4Cidr; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; use nexus_db_queries::context::OpContext; @@ -48,13 +38,9 @@ use omicron_common::api::external::Name; use omicron_common::api::external::NameOrId; use std::collections::HashMap; use std::net::IpAddr; -use std::net::Ipv4Addr; -use std::net::Ipv6Addr; use std::str::FromStr; use uuid::Uuid; -use super::sagas::NEXUS_DPD_TAG; - impl super::Nexus { pub async fn racks_list( &self, @@ -238,9 +224,9 @@ impl super::Nexus { // Currently calling some of the apis directly, but should we be using sagas // going forward via self.run_saga()? Note that self.create_runnable_saga and // self.execute_saga are currently not available within this scope. - info!(self.log, "checking for rack networking configuration"); + info!(self.log, "Checking for Rack Network Configuration"); if let Some(rack_network_config) = &request.rack_network_config { - info!(self.log, "configuring rack networking"); + info!(self.log, "Recording Rack Network Configuration"); let address_lot_name = Name::from_str("initial-infra").map_err(|e| { Error::internal_error(&format!( @@ -366,20 +352,6 @@ impl super::Nexus { .await?; } - let body = Ipv6Entry { - addr: Ipv6Addr::from_str("fd00:99::1").map_err(|e| { - Error::internal_error(&format!( - "failed to parse `fd00:99::1` as `Ipv6Addr`: {e}" - )) - })?, - tag: NEXUS_DPD_TAG.into(), - }; - self.dpd_client.loopback_ipv6_create(&body).await.map_err(|e| { - Error::internal_error(&format!( - "unable to create inital switch loopback address: {e}" - )) - })?; - let name = Name::from_str("default-uplink").map_err(|e| { Error::internal_error(&format!( "unable to use `default-uplink` as `Name`: {e}" @@ -455,59 +427,7 @@ impl super::Nexus { .await?; } - let mut dpd_port_settings = PortSettings { - tag: NEXUS_DPD_TAG.into(), - links: HashMap::new(), - v4_routes: HashMap::new(), - v6_routes: HashMap::new(), - }; - - // TODO handle breakouts - // https://github.com/oxidecomputer/omicron/issues/3062 - let link_id = LinkId(0); - let addr = IpAddr::from_str(&rack_network_config.uplink_ip) - .map_err(|e| { - Error::internal_error(&format!( - "unable to parse rack_network_config.uplink_up as IpAddr: {e}")) - })?; - - let link_settings = LinkSettings { - // TODO Allow user to configure link properties - // https://github.com/oxidecomputer/omicron/issues/3061 - params: LinkCreate { - autoneg: false, - kr: false, - fec: PortFec::None, - speed: PortSpeed::Speed100G, - }, - addrs: vec![addr], - }; - - dpd_port_settings.links.insert(link_id.to_string(), link_settings); - - let port_id: PortId = PortId::from_str(&rack_network_config.uplink_port) - .map_err(|e| Error::internal_error( - &format!("could not use value provided to rack_network_config.uplink_port as PortID: {e}") - ))?; - - let nexthop = Some(Ipv4Addr::from_str(&rack_network_config.gateway_ip) - .map_err(|e| Error::internal_error( - &format!("unable to parse rack_network_config.gateway_ip as Ipv4Addr: {e}") - ))?); - - dpd_port_settings.v4_routes.insert( - Ipv4Cidr { - prefix: Ipv4Addr::from_str("0.0.0.0").unwrap(), - prefix_len: 0, - } - .to_string(), - RouteSettingsV4 { link_id: link_id.0, nexthop }, - ); - - self.dpd_client - .port_settings_apply(&port_id, &dpd_port_settings) - .await - .map_err(|e| Error::internal_error(&format!("unable to apply initial uplink port configuration: {e}")))?; + // TODO - record port speed }; Ok(()) diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 60195920d5..ff7a720484 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -280,6 +280,21 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortSpeed": { + "description": "Switchport Speed options", + "type": "string", + "enum": [ + "speed0_g", + "speed1_g", + "speed10_g", + "speed25_g", + "speed40_g", + "speed50_g", + "speed100_g", + "speed200_g", + "speed400_g" + ] + }, "RackInitializeRequest": { "description": "Configuration for the \"rack setup service\".\n\nThe Rack Setup Service should be responsible for one-time setup actions, such as CockroachDB placement and initialization. Without operator intervention, however, these actions need a way to be automated in our deployment.", "type": "object", @@ -387,6 +402,14 @@ "uplink_port": { "description": "Switchport to use for external connectivity", "type": "string" + }, + "uplink_port_speed": { + "description": "Speed for the Switchport", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] } }, "required": [ @@ -394,7 +417,8 @@ "infra_ip_first", "infra_ip_last", "uplink_ip", - "uplink_port" + "uplink_port", + "uplink_port_speed" ] }, "RecoverySiloConfig": { diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 69227e4406..e903932722 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -2280,6 +2280,21 @@ "PhysicalDiskPutResponse": { "type": "object" }, + "PortSpeed": { + "description": "Switchport Speed options", + "type": "string", + "enum": [ + "speed0_g", + "speed1_g", + "speed10_g", + "speed25_g", + "speed40_g", + "speed50_g", + "speed100_g", + "speed200_g", + "speed400_g" + ] + }, "ProducerEndpoint": { "description": "Information announced by a metric server, used so that clients can contact it and collect available metric data from it.", "type": "object", @@ -2449,6 +2464,14 @@ "uplink_port": { "description": "Switchport to use for external connectivity", "type": "string" + }, + "uplink_port_speed": { + "description": "Speed for the Switchport", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] } }, "required": [ @@ -2456,7 +2479,8 @@ "infra_ip_first", "infra_ip_last", "uplink_ip", - "uplink_port" + "uplink_port", + "uplink_port_speed" ] }, "RecoverySiloConfig": { diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index edb6319dee..81b0745bd1 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -74,11 +74,18 @@ use crate::rack_setup::plan::sled::{ use crate::storage_manager::StorageResources; use camino::Utf8PathBuf; use ddm_admin_client::{Client as DdmAdminClient, DdmError}; +use dpd_client::types::{ + LinkCreate, LinkId, LinkSettings, PortFec, PortId, PortSettings, + RouteSettingsV4, +}; +use dpd_client::Client as DpdClient; +use dpd_client::Ipv4Cidr; use internal_dns::resolver::{DnsError, Resolver as DnsResolver}; use internal_dns::ServiceName; use nexus_client::{ types as NexusTypes, Client as NexusClient, Error as NexusError, }; +use omicron_common::address::Ipv6Subnet; use omicron_common::address::{ get_sled_address, CRUCIBLE_PANTRY_PORT, DENDRITE_PORT, NEXUS_INTERNAL_PORT, NTP_PORT, OXIMETER_PORT, @@ -94,9 +101,13 @@ use sled_hardware::underlay::BootstrapInterface; use slog::Logger; use std::collections::{HashMap, HashSet}; use std::iter; +use std::net::IpAddr; +use std::net::Ipv4Addr; use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use thiserror::Error; +static BOUNDARY_SERVICES_ADDR: &str = "fd00:99::1"; + /// Describes errors which may occur while operating the setup service. #[derive(Error, Debug)] pub enum SetupServiceError { @@ -142,6 +153,12 @@ pub enum SetupServiceError { #[error("Failed to access DNS servers: {0}")] Dns(#[from] DnsError), + + #[error("Error during request to Dendrite: {0}")] + Dendrite(String), + + #[error("Error during DNS lookup: {0}")] + DnsResolver(internal_dns::resolver::ResolveError), } // The workload / information allocated to a single sled. @@ -759,6 +776,7 @@ impl ServiceInner { infra_ip_last: config.infra_ip_last.clone(), uplink_ip: config.uplink_ip.clone(), uplink_port: config.uplink_port.clone(), + uplink_port_speed: config.uplink_port_speed.clone().into(), }; Some(value) } @@ -988,6 +1006,119 @@ impl ServiceInner { // DNS configuration to the internal DNS servers. self.initialize_dns(&service_plan).await?; + // Initialize rack network before NTP comes online, otherwise boundary + // services will not be available and NTP will fail to sync + info!(self.log, "Checking for Rack Network Configuration"); + if let Some(rack_network_config) = &config.rack_network_config { + info!(self.log, "Initializing Rack Network"); + info!(self.log, "Looking up address for Dendrite"); + let resolver = DnsResolver::new_from_subnet( + self.log.new(o!("component" => "DnsResolver")), + config.az_subnet(), + ) + .expect("Failed to create DNS resolver"); + + let dpd_addr = resolver + .lookup_socket_v6(internal_dns::ServiceName::Dendrite) + .await + .map_err(|e| SetupServiceError::DnsResolver(e))?; + + let dpd = DpdClient::new( + &format!("http://[{}]:{}", dpd_addr.ip(), dpd_addr.port()), + dpd_client::ClientState { + tag: "sled-agent".to_string(), + log: self.log.new(o!( + "component" => "DpdClient" + )), + }, + ); + + info!(self.log, "Building Rack Network Configuration"); + let body = dpd_client::types::Ipv6Entry { + addr: BOUNDARY_SERVICES_ADDR.parse().map_err(|e| { + SetupServiceError::BadConfig(format!( + "failed to parse `BOUNDARY_SERVICES_ADDR` as `Ipv6Addr`: {e}" + )) + })?, + tag: "rss".into(), + }; + + let mut dpd_port_settings = PortSettings { + tag: "rss".into(), + links: HashMap::new(), + v4_routes: HashMap::new(), + v6_routes: HashMap::new(), + }; + + // TODO handle breakouts + // https://github.com/oxidecomputer/omicron/issues/3062 + let link_id = LinkId(0); + let addr: IpAddr = rack_network_config.uplink_ip.parse() + .map_err(|e| { + SetupServiceError::BadConfig(format!( + "unable to parse rack_network_config.uplink_up as IpAddr: {e}")) + })?; + + let link_settings = LinkSettings { + // TODO Allow user to configure link properties + // https://github.com/oxidecomputer/omicron/issues/3061 + params: LinkCreate { + autoneg: false, + kr: false, + fec: PortFec::None, + speed: rack_network_config.uplink_port_speed.clone().into(), + }, + addrs: vec![addr], + }; + + dpd_port_settings.links.insert(link_id.to_string(), link_settings); + + let port_id: PortId = rack_network_config.uplink_port.parse() + .map_err(|e| SetupServiceError::BadConfig( + format!("could not use value provided to rack_network_config.uplink_port as PortID: {e}") + ))?; + + let nexthop: Option = Some(rack_network_config.gateway_ip.parse() + .map_err(|e| SetupServiceError::BadConfig( + format!("unable to parse rack_network_config.gateway_ip as Ipv4Addr: {e}") + ))?); + + dpd_port_settings.v4_routes.insert( + Ipv4Cidr { prefix: "0.0.0.0".parse().unwrap(), prefix_len: 0 } + .to_string(), + RouteSettingsV4 { link_id: link_id.0, nexthop }, + ); + + info!(self.log, "Waiting for dendrite to come online"); + while dpd.dpd_uptime().await.is_err() { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + + info!(self.log, "Configuring boundary services loopback address on switch"; "config" => format!("{body:#?}")); + dpd.loopback_ipv6_create(&body).await.map_err(|e| { + SetupServiceError::Dendrite(format!( + "unable to create inital switch loopback address: {e}" + )) + })?; + + info!(self.log, "Configuring default uplink on switch"; "config" => format!("{dpd_port_settings:#?}")); + dpd.port_settings_apply(&port_id, &dpd_port_settings) + .await + .map_err(|e| { + SetupServiceError::Dendrite(format!( + "unable to apply initial uplink port configuration: {e}" + )) + })?; + + info!(self.log, "advertising boundary services loopback address"); + let mut ddmd_addr = dpd_addr; + ddmd_addr.set_port(8000); + let ddmd_client = DdmAdminClient::new(&self.log, ddmd_addr)?; + ddmd_client.advertise_prefix(Ipv6Subnet::new( + BOUNDARY_SERVICES_ADDR.parse().unwrap(), + )); + } + // Next start up the NTP services. // Note we also specify internal DNS services again because it // can ony be additive. diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index e57be11898..d28de70034 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -63,4 +63,5 @@ gateway_ip = "192.168.1.199" infra_ip_first = "192.168.1.30" infra_ip_last = "192.168.1.50" uplink_port = "qsfp0" +uplink_port_speed = "40G" uplink_ip = "192.168.1.32" From dcf246765da354d5c9ff1691605cf5a55024f31b Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Fri, 2 Jun 2023 02:44:38 +0000 Subject: [PATCH 2/3] PR fixes --- common/src/api/internal/shared.rs | 21 +++++++++++++++++++ nexus-client/src/lib.rs | 16 +++++++++++++++ nexus/src/app/rack.rs | 3 ++- openapi/bootstrap-agent.json | 18 ++++++++++++++++ openapi/nexus-internal.json | 18 ++++++++++++++++ sled-agent/src/rack_setup/service.rs | 25 ++++++++++++++++------- smf/sled-agent/non-gimlet/config-rss.toml | 1 + 7 files changed, 94 insertions(+), 8 deletions(-) diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 680be80ed3..b5c1ee4218 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -75,6 +75,8 @@ pub struct RackNetworkConfig { pub uplink_port: String, /// Speed for the Switchport pub uplink_port_speed: PortSpeed, + /// Forward Error Correction setting for the uplink port + pub uplink_port_fec: PortFec, /// IP Address to apply to switchport (must be in infra_ip pool) pub uplink_ip: String, } @@ -118,3 +120,22 @@ impl From for dpd_client::types::PortSpeed { } } } + +/// Switchport FEC options +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum PortFec { + Firecode, + None, + Rs, +} + +impl From for dpd_client::types::PortFec { + fn from(value: PortFec) -> Self { + match value { + PortFec::Firecode => dpd_client::types::PortFec::Firecode, + PortFec::None => dpd_client::types::PortFec::None, + PortFec::Rs => dpd_client::types::PortFec::Rs, + } + } +} diff --git a/nexus-client/src/lib.rs b/nexus-client/src/lib.rs index 234284acbc..e82d48d3c0 100644 --- a/nexus-client/src/lib.rs +++ b/nexus-client/src/lib.rs @@ -308,3 +308,19 @@ impl From } } } + +impl From for types::PortFec { + fn from(value: omicron_common::api::internal::shared::PortFec) -> Self { + match value { + omicron_common::api::internal::shared::PortFec::Firecode => { + types::PortFec::Firecode + } + omicron_common::api::internal::shared::PortFec::None => { + types::PortFec::None + } + omicron_common::api::internal::shared::PortFec::Rs => { + types::PortFec::Rs + } + } + } +} diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 8fa361a937..61e437930c 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -427,7 +427,8 @@ impl super::Nexus { .await?; } - // TODO - record port speed + // TODO - https://github.com/oxidecomputer/omicron/issues/3277 + // record port speed }; Ok(()) diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index ff7a720484..7be462a440 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -280,6 +280,15 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortFec": { + "description": "Switchport FEC options", + "type": "string", + "enum": [ + "firecode", + "none", + "rs" + ] + }, "PortSpeed": { "description": "Switchport Speed options", "type": "string", @@ -403,6 +412,14 @@ "description": "Switchport to use for external connectivity", "type": "string" }, + "uplink_port_fec": { + "description": "Forward Error Correction setting for the uplink port", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, "uplink_port_speed": { "description": "Speed for the Switchport", "allOf": [ @@ -418,6 +435,7 @@ "infra_ip_last", "uplink_ip", "uplink_port", + "uplink_port_fec", "uplink_port_speed" ] }, diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index e903932722..c01745d5e4 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -2280,6 +2280,15 @@ "PhysicalDiskPutResponse": { "type": "object" }, + "PortFec": { + "description": "Switchport FEC options", + "type": "string", + "enum": [ + "firecode", + "none", + "rs" + ] + }, "PortSpeed": { "description": "Switchport Speed options", "type": "string", @@ -2465,6 +2474,14 @@ "description": "Switchport to use for external connectivity", "type": "string" }, + "uplink_port_fec": { + "description": "Forward Error Correction setting for the uplink port", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, "uplink_port_speed": { "description": "Speed for the Switchport", "allOf": [ @@ -2480,6 +2497,7 @@ "infra_ip_last", "uplink_ip", "uplink_port", + "uplink_port_fec", "uplink_port_speed" ] }, diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 81b0745bd1..d56dce48bd 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -158,7 +158,7 @@ pub enum SetupServiceError { Dendrite(String), #[error("Error during DNS lookup: {0}")] - DnsResolver(internal_dns::resolver::ResolveError), + DnsResolver(#[from] internal_dns::resolver::ResolveError), } // The workload / information allocated to a single sled. @@ -777,6 +777,7 @@ impl ServiceInner { uplink_ip: config.uplink_ip.clone(), uplink_port: config.uplink_port.clone(), uplink_port_speed: config.uplink_port_speed.clone().into(), + uplink_port_fec: config.uplink_port_fec.clone().into(), }; Some(value) } @@ -1015,13 +1016,11 @@ impl ServiceInner { let resolver = DnsResolver::new_from_subnet( self.log.new(o!("component" => "DnsResolver")), config.az_subnet(), - ) - .expect("Failed to create DNS resolver"); + )?; let dpd_addr = resolver .lookup_socket_v6(internal_dns::ServiceName::Dendrite) - .await - .map_err(|e| SetupServiceError::DnsResolver(e))?; + .await?; let dpd = DpdClient::new( &format!("http://[{}]:{}", dpd_addr.ip(), dpd_addr.port()), @@ -1034,6 +1033,8 @@ impl ServiceInner { ); info!(self.log, "Building Rack Network Configuration"); + // TODO - https://github.com/oxidecomputer/omicron/issues/3278 + // dynamically determine where boundary services address should be configured let body = dpd_client::types::Ipv6Entry { addr: BOUNDARY_SERVICES_ADDR.parse().map_err(|e| { SetupServiceError::BadConfig(format!( @@ -1089,8 +1090,18 @@ impl ServiceInner { RouteSettingsV4 { link_id: link_id.0, nexthop }, ); - info!(self.log, "Waiting for dendrite to come online"); - while dpd.dpd_uptime().await.is_err() { + loop { + info!(self.log, "Checking dendrite uptime"); + match dpd.dpd_uptime().await { + Ok(uptime) => { + info!(self.log, "Dendrite online"; "uptime" => uptime.to_string()); + break; + } + Err(e) => { + info!(self.log, "Unable to check Dendrite uptime"; "reason" => format!("{e}")); + } + } + info!(self.log, "Waiting for dendrite to come online"); tokio::time::sleep(std::time::Duration::from_secs(2)).await; } diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index d28de70034..98ed902a73 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -64,4 +64,5 @@ infra_ip_first = "192.168.1.30" infra_ip_last = "192.168.1.50" uplink_port = "qsfp0" uplink_port_speed = "40G" +uplink_port_fec="none" uplink_ip = "192.168.1.32" From db8fabb54a8026887420fa421b30793799f26a32 Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Fri, 2 Jun 2023 03:00:00 +0000 Subject: [PATCH 3/3] actually plumb the config through --- sled-agent/src/rack_setup/service.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index d56dce48bd..04bd940022 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -75,8 +75,7 @@ use crate::storage_manager::StorageResources; use camino::Utf8PathBuf; use ddm_admin_client::{Client as DdmAdminClient, DdmError}; use dpd_client::types::{ - LinkCreate, LinkId, LinkSettings, PortFec, PortId, PortSettings, - RouteSettingsV4, + LinkCreate, LinkId, LinkSettings, PortId, PortSettings, RouteSettingsV4, }; use dpd_client::Client as DpdClient; use dpd_client::Ipv4Cidr; @@ -1066,7 +1065,7 @@ impl ServiceInner { params: LinkCreate { autoneg: false, kr: false, - fec: PortFec::None, + fec: rack_network_config.uplink_port_fec.clone().into(), speed: rack_network_config.uplink_port_speed.clone().into(), }, addrs: vec![addr],