diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index ec9fed5f33..a4a7487f91 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -144,6 +144,8 @@ table! { lldp_service_config_id -> Uuid, link_name -> Text, mtu -> Int4, + fec -> crate::SwitchLinkFecEnum, + speed -> crate::SwitchLinkSpeedEnum, } } @@ -203,6 +205,11 @@ table! { bgp_config_id -> Uuid, interface_name -> Text, addr -> Inet, + hold_time -> Int8, + idle_hold_time -> Int8, + delay_open -> Int8, + connect_retry -> Int8, + keepalive -> Int8, } } diff --git a/nexus/db-model/src/switch_port.rs b/nexus/db-model/src/switch_port.rs index 086508e0e0..f6df50ef97 100644 --- a/nexus/db-model/src/switch_port.rs +++ b/nexus/db-model/src/switch_port.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::impl_enum_type; use crate::schema::{ lldp_config, lldp_service_config, switch_port, switch_port_settings, switch_port_settings_address_config, switch_port_settings_bgp_peer_config, @@ -11,12 +10,14 @@ use crate::schema::{ switch_port_settings_port_config, switch_port_settings_route_config, }; use crate::SqlU16; +use crate::{impl_enum_type, SqlU32}; use db_macros::Resource; use diesel::AsChangeset; use ipnetwork::IpNetwork; use nexus_types::external_api::params; use nexus_types::identity::Resource; use omicron_common::api::external; +use omicron_common::api::internal::shared::{PortFec, PortSpeed}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -43,6 +44,110 @@ impl_enum_type!( Sfp28x4 => b"Sfp28x4" ); +impl_enum_type!( + #[derive(SqlType, Debug, Clone, Copy)] + #[diesel(postgres_type(name = "switch_link_fec"))] + pub struct SwitchLinkFecEnum; + + #[derive( + Clone, + Copy, + Debug, + AsExpression, + FromSqlRow, + PartialEq, + Serialize, + Deserialize + )] + #[diesel(sql_type = SwitchLinkFecEnum)] + pub enum SwitchLinkFec; + + Firecode => b"Firecode" + None => b"None" + Rs => b"Rs" +); + +impl_enum_type!( + #[derive(SqlType, Debug, Clone, Copy)] + #[diesel(postgres_type(name = "switch_link_speed"))] + pub struct SwitchLinkSpeedEnum; + + #[derive( + Clone, + Copy, + Debug, + AsExpression, + FromSqlRow, + PartialEq, + Serialize, + Deserialize + )] + #[diesel(sql_type = SwitchLinkSpeedEnum)] + pub enum SwitchLinkSpeed; + + Speed0G => b"0G" + Speed1G => b"1G" + Speed10G => b"10G" + Speed25G => b"25G" + Speed40G => b"40G" + Speed50G => b"50G" + Speed100G => b"100G" + Speed200G => b"200G" + Speed400G => b"400G" +); + +impl From for PortFec { + fn from(value: SwitchLinkFec) -> Self { + match value { + SwitchLinkFec::Firecode => PortFec::Firecode, + SwitchLinkFec::None => PortFec::None, + SwitchLinkFec::Rs => PortFec::Rs, + } + } +} + +impl From for SwitchLinkFec { + fn from(value: params::LinkFec) -> Self { + match value { + params::LinkFec::Firecode => SwitchLinkFec::Firecode, + params::LinkFec::None => SwitchLinkFec::None, + params::LinkFec::Rs => SwitchLinkFec::Rs, + } + } +} + +impl From for PortSpeed { + fn from(value: SwitchLinkSpeed) -> Self { + match value { + SwitchLinkSpeed::Speed0G => PortSpeed::Speed0G, + SwitchLinkSpeed::Speed1G => PortSpeed::Speed1G, + SwitchLinkSpeed::Speed10G => PortSpeed::Speed10G, + SwitchLinkSpeed::Speed25G => PortSpeed::Speed25G, + SwitchLinkSpeed::Speed40G => PortSpeed::Speed40G, + SwitchLinkSpeed::Speed50G => PortSpeed::Speed50G, + SwitchLinkSpeed::Speed100G => PortSpeed::Speed100G, + SwitchLinkSpeed::Speed200G => PortSpeed::Speed200G, + SwitchLinkSpeed::Speed400G => PortSpeed::Speed400G, + } + } +} + +impl From for SwitchLinkSpeed { + fn from(value: params::LinkSpeed) -> Self { + match value { + params::LinkSpeed::Speed0G => SwitchLinkSpeed::Speed0G, + params::LinkSpeed::Speed1G => SwitchLinkSpeed::Speed1G, + params::LinkSpeed::Speed10G => SwitchLinkSpeed::Speed10G, + params::LinkSpeed::Speed25G => SwitchLinkSpeed::Speed25G, + params::LinkSpeed::Speed40G => SwitchLinkSpeed::Speed40G, + params::LinkSpeed::Speed50G => SwitchLinkSpeed::Speed50G, + params::LinkSpeed::Speed100G => SwitchLinkSpeed::Speed100G, + params::LinkSpeed::Speed200G => SwitchLinkSpeed::Speed200G, + params::LinkSpeed::Speed400G => SwitchLinkSpeed::Speed400G, + } + } +} + impl From for SwitchPortGeometry { fn from(g: params::SwitchPortGeometry) -> Self { match g { @@ -241,6 +346,8 @@ pub struct SwitchPortLinkConfig { pub lldp_service_config_id: Uuid, pub link_name: String, pub mtu: SqlU16, + pub fec: SwitchLinkFec, + pub speed: SwitchLinkSpeed, } impl SwitchPortLinkConfig { @@ -249,11 +356,15 @@ impl SwitchPortLinkConfig { lldp_service_config_id: Uuid, link_name: String, mtu: u16, + fec: SwitchLinkFec, + speed: SwitchLinkSpeed, ) -> Self { Self { port_settings_id, lldp_service_config_id, link_name, + fec, + speed, mtu: mtu.into(), } } @@ -442,16 +553,37 @@ pub struct SwitchPortBgpPeerConfig { pub bgp_config_id: Uuid, pub interface_name: String, pub addr: IpNetwork, + pub hold_time: SqlU32, + pub idle_hold_time: SqlU32, + pub delay_open: SqlU32, + pub connect_retry: SqlU32, + pub keepalive: SqlU32, } impl SwitchPortBgpPeerConfig { + #[allow(clippy::too_many_arguments)] pub fn new( port_settings_id: Uuid, bgp_config_id: Uuid, interface_name: String, addr: IpNetwork, + hold_time: SqlU32, + idle_hold_time: SqlU32, + delay_open: SqlU32, + connect_retry: SqlU32, + keepalive: SqlU32, ) -> Self { - Self { port_settings_id, bgp_config_id, interface_name, addr } + Self { + port_settings_id, + bgp_config_id, + interface_name, + addr, + hold_time, + idle_hold_time, + delay_open, + connect_retry, + keepalive, + } } } diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index 9d28a31013..f2126bd968 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -227,6 +227,8 @@ impl DataStore { lldp_svc_config.id, link_name.clone(), c.mtu, + c.fec.into(), + c.speed.into(), )); } result.link_lldp = @@ -298,28 +300,6 @@ impl DataStore { let mut bgp_peer_config = Vec::new(); for (interface_name, p) in ¶ms.bgp_peers { - /* XXX ANNOUNCE - use db::schema::bgp_announce_set; - let announce_set_id = match &p.bgp_announce_set { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - bgp_announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::time_deleted.is_null()) - .filter(bgp_announce_set::name.eq(name)) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpAnnounceSetNotFound, - ) - })? - } - }; - */ - use db::schema::bgp_config; let bgp_config_id = match &p.bgp_config { NameOrId::Id(id) => *id, @@ -345,6 +325,11 @@ impl DataStore { bgp_config_id, interface_name.clone(), p.addr.into(), + p.hold_time.into(), + p.idle_hold_time.into(), + p.delay_open.into(), + p.connect_retry.into(), + p.keepalive.into(), )); } diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index a77b43965f..e2a9ef6e79 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -19,14 +19,14 @@ use internal_dns::ServiceName; use ipnetwork::IpNetwork; use mg_admin_client::types::Prefix4; use mg_admin_client::types::{ApplyRequest, BgpPeerConfig, BgpRoute}; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::UpdatePrecondition; use nexus_db_queries::{authn, db}; use nexus_types::external_api::params; use omicron_common::api::external::{self, NameOrId}; use omicron_common::api::internal::shared::{ - ParseSwitchLocationError, PortFec as OmicronPortFec, - PortSpeed as OmicronPortSpeed, SwitchLocation, + ParseSwitchLocationError, SwitchLocation, }; use serde::{Deserialize, Serialize}; use sled_agent_client::types::PortConfigV1; @@ -43,6 +43,11 @@ use std::sync::Arc; use steno::ActionError; use uuid::Uuid; +// This is more of an implementation detail of the BGP implementation. It +// defines the maximum time the peering engine will wait for external messages +// before breaking to check for shutdown conditions. +const BGP_SESSION_RESOLUTION: u64 = 100; + // switch port settings apply saga: input parameters #[derive(Debug, Deserialize, Serialize)] @@ -418,14 +423,14 @@ pub(crate) async fn ensure_switch_port_bgp_settings( let bpc = BgpPeerConfig { asn: *config.asn, - name: format!("{}", peer.addr.ip()), //TODO(ry)(user defined name) + name: format!("{}", peer.addr.ip()), //TODO user defined name? host: format!("{}:179", peer.addr.ip()), - hold_time: 6, //TODO(ry)(hardocde) - idle_hold_time: 6, //TODO(ry)(hardocde) - delay_open: 0, //TODO(ry)(hardocde) - connect_retry: 0, //TODO(ry)(hardcode) - keepalive: 3, //TODO(ry)(hardcode) - resolution: 100, //TODO(ry)(hardcode) + hold_time: peer.hold_time.0.into(), + idle_hold_time: peer.idle_hold_time.0.into(), + delay_open: peer.delay_open.0.into(), + connect_retry: peer.connect_retry.0.into(), + keepalive: peer.keepalive.0.into(), + resolution: BGP_SESSION_RESOLUTION, routes: vec![BgpRoute { nexthop, prefixes }], }; @@ -888,8 +893,18 @@ pub(crate) async fn bootstore_update( addresses: settings.addresses.iter().map(|a| a.address).collect(), switch: switch_location, port: switch_port_name.into(), - uplink_port_fec: OmicronPortFec::None, //TODO hardcode - uplink_port_speed: OmicronPortSpeed::Speed100G, //TODO hardcode + uplink_port_fec: settings + .links + .get(0) + .map(|l| l.fec) + .unwrap_or(SwitchLinkFec::None) + .into(), + uplink_port_speed: settings + .links + .get(0) + .map(|l| l.speed) + .unwrap_or(SwitchLinkSpeed::Speed100G) + .into(), bgp_peers: peer_info .iter() .filter_map(|(p, asn)| { diff --git a/nexus/src/app/switch_port.rs b/nexus/src/app/switch_port.rs index 537f52d5df..74dcd90e51 100644 --- a/nexus/src/app/switch_port.rs +++ b/nexus/src/app/switch_port.rs @@ -6,6 +6,7 @@ use crate::app::sagas; use crate::external_api::params; use db::datastore::SwitchPortSettingsCombinedResult; use ipnetwork::IpNetwork; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::authn; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; @@ -394,12 +395,18 @@ impl super::Nexus { .collect(), switch: port.switch_location.parse().unwrap(), port: port.port_name.clone(), - //TODO hardcode - uplink_port_fec: - omicron_common::api::internal::shared::PortFec::None, - //TODO hardcode - uplink_port_speed: - omicron_common::api::internal::shared::PortSpeed::Speed100G, + uplink_port_fec: info + .links + .get(0) //TODO breakout support + .map(|l| l.fec) + .unwrap_or(SwitchLinkFec::None) + .into(), + uplink_port_speed: info + .links + .get(0) //TODO breakout support + .map(|l| l.speed) + .unwrap_or(SwitchLinkSpeed::Speed100G) + .into(), }; rack_net_config.ports.push(p); diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index b6b9b193e5..fada45694d 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -11,9 +11,9 @@ use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params::{ Address, AddressConfig, AddressLotBlockCreate, AddressLotCreate, BgpAnnounceSetCreate, BgpAnnouncementCreate, BgpConfigCreate, - BgpPeerConfig, LinkConfig, LldpServiceConfig, Route, RouteConfig, - SwitchInterfaceConfig, SwitchInterfaceKind, SwitchPortApplySettings, - SwitchPortSettingsCreate, + BgpPeerConfig, LinkConfig, LinkFec, LinkSpeed, LldpServiceConfig, Route, + RouteConfig, SwitchInterfaceConfig, SwitchInterfaceKind, + SwitchPortApplySettings, SwitchPortSettingsCreate, }; use nexus_types::external_api::views::Rack; use omicron_common::api::external::{ @@ -112,6 +112,8 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { LinkConfig { mtu: 4700, lldp: LldpServiceConfig { enabled: false, lldp_config: None }, + fec: LinkFec::None, + speed: LinkSpeed::Speed100G, }, ); // interfaces @@ -250,6 +252,11 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { bgp_announce_set: NameOrId::Name("instances".parse().unwrap()), //TODO interface_name: "phy0".to_string(), addr: "1.2.3.4".parse().unwrap(), + hold_time: 6, + idle_hold_time: 6, + delay_open: 0, + connect_retry: 3, + keepalive: 2, }, ); let _created: SwitchPortSettingsView = NexusRequest::objects_post( diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index fa25a576a7..a0169ae777 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -1325,6 +1325,42 @@ pub enum SwitchPortGeometry { Sfp28x4, } +/// The forward error correction mode of a link. +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LinkFec { + /// Firecode foward error correction. + Firecode, + /// No forward error correction. + None, + /// Reed-Solomon forward error correction. + Rs, +} + +/// The speed of a link. +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LinkSpeed { + /// Zero gigabits per second. + Speed0G, + /// 1 gigabit per second. + Speed1G, + /// 10 gigabits per second. + Speed10G, + /// 25 gigabits per second. + Speed25G, + /// 40 gigabits per second. + Speed40G, + /// 50 gigabits per second. + Speed50G, + /// 100 gigabits per second. + Speed100G, + /// 200 gigabits per second. + Speed200G, + /// 400 gigabits per second. + Speed400G, +} + /// Switch link configuration. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct LinkConfig { @@ -1333,6 +1369,12 @@ pub struct LinkConfig { /// The link-layer discovery protocol (LLDP) configuration for the link. pub lldp: LldpServiceConfig, + + /// The forward error correction mode of the link. + pub fec: LinkFec, + + /// The speed of the link. + pub speed: LinkSpeed, } /// The LLDP configuration associated with a port. LLDP may be either enabled or @@ -1441,6 +1483,23 @@ pub struct BgpPeerConfig { /// The address of the host to peer with. pub addr: IpAddr, + + /// How long to hold peer connections between keppalives (seconds). + pub hold_time: u32, + + /// How long to hold a peer in idle before attempting a new session + /// (seconds). + pub idle_hold_time: u32, + + /// How long to delay sending an open request after establishing a TCP + /// session (seconds). + pub delay_open: u32, + + /// How long to to wait between TCP connection retries (seconds). + pub connect_retry: u32, + + /// How often to send keepalive requests (seconds). + pub keepalive: u32, } /// Parameters for creating a named set of BGP announcements. diff --git a/openapi/nexus.json b/openapi/nexus.json index 2ff907eb67..456f2aebd6 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -8420,16 +8420,51 @@ } ] }, + "connect_retry": { + "description": "How long to to wait between TCP connection retries (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "delay_open": { + "description": "How long to delay sending an open request after establishing a TCP session (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "hold_time": { + "description": "How long to hold peer connections between keppalives (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "idle_hold_time": { + "description": "How long to hold a peer in idle before attempting a new session (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "interface_name": { "description": "The name of interface to peer on. This is relative to the port configuration this BGP peer configuration is a part of. For example this value could be phy0 to refer to a primary physical interface. Or it could be vlan47 to refer to a VLAN interface.", "type": "string" + }, + "keepalive": { + "description": "How often to send keepalive requests (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 } }, "required": [ "addr", "bgp_announce_set", "bgp_config", - "interface_name" + "connect_retry", + "delay_open", + "hold_time", + "idle_hold_time", + "interface_name", + "keepalive" ] }, "BgpPeerState": { @@ -12433,6 +12468,14 @@ "description": "Switch link configuration.", "type": "object", "properties": { + "fec": { + "description": "The forward error correction mode of the link.", + "allOf": [ + { + "$ref": "#/components/schemas/LinkFec" + } + ] + }, "lldp": { "description": "The link-layer discovery protocol (LLDP) configuration for the link.", "allOf": [ @@ -12446,11 +12489,115 @@ "type": "integer", "format": "uint16", "minimum": 0 + }, + "speed": { + "description": "The speed of the link.", + "allOf": [ + { + "$ref": "#/components/schemas/LinkSpeed" + } + ] } }, "required": [ + "fec", "lldp", - "mtu" + "mtu", + "speed" + ] + }, + "LinkFec": { + "description": "The forward error correction mode of a link.", + "oneOf": [ + { + "description": "Firecode foward error correction.", + "type": "string", + "enum": [ + "firecode" + ] + }, + { + "description": "No forward error correction.", + "type": "string", + "enum": [ + "none" + ] + }, + { + "description": "Reed-Solomon forward error correction.", + "type": "string", + "enum": [ + "rs" + ] + } + ] + }, + "LinkSpeed": { + "description": "The speed of a link.", + "oneOf": [ + { + "description": "Zero gigabits per second.", + "type": "string", + "enum": [ + "speed0_g" + ] + }, + { + "description": "1 gigabit per second.", + "type": "string", + "enum": [ + "speed1_g" + ] + }, + { + "description": "10 gigabits per second.", + "type": "string", + "enum": [ + "speed10_g" + ] + }, + { + "description": "25 gigabits per second.", + "type": "string", + "enum": [ + "speed25_g" + ] + }, + { + "description": "40 gigabits per second.", + "type": "string", + "enum": [ + "speed40_g" + ] + }, + { + "description": "50 gigabits per second.", + "type": "string", + "enum": [ + "speed50_g" + ] + }, + { + "description": "100 gigabits per second.", + "type": "string", + "enum": [ + "speed100_g" + ] + }, + { + "description": "200 gigabits per second.", + "type": "string", + "enum": [ + "speed200_g" + ] + }, + { + "description": "400 gigabits per second.", + "type": "string", + "enum": [ + "speed400_g" + ] + } ] }, "LldpServiceConfig": { diff --git a/schema/crdb/8.0.0/up3.sql b/schema/crdb/8.0.0/up3.sql new file mode 100644 index 0000000000..aa165606e2 --- /dev/null +++ b/schema/crdb/8.0.0/up3.sql @@ -0,0 +1,26 @@ +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM ( + 'Firecode', + 'None', + 'Rs' +); + +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( + '0G', + '1G', + '10G', + '25G', + '40G', + '50G', + '100G', + '200G', + '400G' +); + +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; + +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS hold_time INT8; +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS idle_hold_time INT8; +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS delay_open INT8; +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS connect_retry INT8; +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS keepalive INT8; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 024ff24638..31894a36a2 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -2445,6 +2445,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_bgp_peer_config ( bgp_config_id UUID NOT NULL, interface_name TEXT, addr INET, + hold_time INT8, + idle_hold_time INT8, + delay_open INT8, + connect_retry INT8, + keepalive INT8, /* TODO https://github.com/oxidecomputer/omicron/issues/3013 */ PRIMARY KEY (port_settings_id, interface_name, addr) @@ -2591,4 +2596,25 @@ FROM WHERE instance.time_deleted IS NULL AND vmm.time_deleted IS NULL; +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM ( + 'Firecode', + 'None', + 'Rs' +); + +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( + '0G', + '1G', + '10G', + '25G', + '40G', + '50G', + '100G', + '200G', + '400G' +); + +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; + COMMIT;