From 978cf08adfc1fb6ec4fdf39814d625eba46447eb Mon Sep 17 00:00:00 2001 From: Ryan Goodfellow Date: Tue, 23 Jan 2024 14:29:33 -0800 Subject: [PATCH] updates for tunnel routing --- .github/buildomat/jobs/deploy.sh | 21 +++- Cargo.lock | 19 ++- Cargo.toml | 4 +- clients/ddm-admin-client/src/lib.rs | 18 ++- illumos-utils/src/opte/mod.rs | 20 ---- illumos-utils/src/opte/port_manager.rs | 3 - .../app/sagas/switch_port_settings_apply.rs | 84 +++++++++++++ .../app/sagas/switch_port_settings_clear.rs | 112 +++++++++++++++++- .../app/sagas/switch_port_settings_common.rs | 41 +------ package-manifest.toml | 12 +- sled-agent/src/bootstrap/early_networking.rs | 69 +++++------ smf/sled-agent/non-gimlet/config-rss.toml | 2 +- tools/ci_check_opte_ver.sh | 5 + tools/install_opte.sh | 10 ++ tools/maghemite_ddm_openapi_version | 4 +- tools/maghemite_mg_openapi_version | 4 +- tools/maghemite_mgd_checksums | 4 +- tools/opte_version | 2 +- tools/opte_version_override | 5 + 19 files changed, 320 insertions(+), 119 deletions(-) create mode 100644 tools/opte_version_override diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index f4f1e0a999..e69cfb0078 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -2,7 +2,7 @@ #: #: name = "helios / deploy" #: variety = "basic" -#: target = "lab-2.0-opte-0.27" +#: target = "lab-2.0-opte-0.28" #: output_rules = [ #: "%/var/svc/log/oxide-sled-agent:default.log*", #: "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*", @@ -33,6 +33,9 @@ _exit_trap() { local status=$? [[ $status -eq 0 ]] && exit 0 + # XXX paranoia + pfexec cp /tmp/opteadm /opt/oxide/opte/bin/opteadm + set +o errexit set -o xtrace banner evidence @@ -50,6 +53,7 @@ _exit_trap() { standalone \ dump-state pfexec /opt/oxide/opte/bin/opteadm list-ports + pfexec /opt/oxide/opte/bin/opteadm dump-v2b z_swadm link ls z_swadm addr list z_swadm route list @@ -97,6 +101,19 @@ z_swadm () { pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm $@ } +# XXX remove. This is just to test against a development branch of OPTE in CI. +set +x +OPTE_COMMIT="73d4669ea213d0b7aca35c4babb6fd09ed51d29e" +curl -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde +pfexec rem_drv xde || true +pfexec mv xde /kernel/drv/amd64/xde +pfexec add_drv xde || true +curl -sSfOL https://buildomat.eng.oxide.computer/wg/0/artefact/01HM09S4M15WNXB2B2MX8R1GBT/yLalJU5vT4S4IEpwSeY4hPuspxw3JcINokZmlfNU14npHkzG/01HM09SJ2RQSFGW7MVKC9JKZ8D/01HM0A58D888AJ7YP6N1Q6T6ZD/opteadm +chmod +x opteadm +cp opteadm /tmp/opteadm +pfexec mv opteadm /opt/oxide/opte/bin/opteadm +set -x + # # XXX work around 14537 (UFS should not allow directories to be unlinked) which # is probably not yet fixed in xde branch? Once the xde branch merges from @@ -236,7 +253,7 @@ infra_ip_last = \"$UPLINK_IP\" /^routes/c\\ routes = \\[{nexthop = \"$GATEWAY_IP\", destination = \"0.0.0.0/0\"}\\] /^addresses/c\\ -addresses = \\[\"$UPLINK_IP/32\"\\] +addresses = \\[\"$UPLINK_IP/24\"\\] } " pkg/config-rss.toml diff -u pkg/config-rss.toml{~,} || true diff --git a/Cargo.lock b/Cargo.lock index a2d9601a38..b5cf705cb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3193,7 +3193,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=dd2b7b0306d3f01fa09170b8884d402209e49244#dd2b7b0306d3f01fa09170b8884d402209e49244" +source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1" [[package]] name = "illumos-utils" @@ -3601,7 +3601,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=dd2b7b0306d3f01fa09170b8884d402209e49244#dd2b7b0306d3f01fa09170b8884d402209e49244" +source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1" dependencies = [ "quote", "syn 2.0.46", @@ -5342,7 +5342,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=dd2b7b0306d3f01fa09170b8884d402209e49244#dd2b7b0306d3f01fa09170b8884d402209e49244" +source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1" dependencies = [ "cfg-if", "dyn-clone", @@ -5358,7 +5358,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=dd2b7b0306d3f01fa09170b8884d402209e49244#dd2b7b0306d3f01fa09170b8884d402209e49244" +source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -5370,7 +5370,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=dd2b7b0306d3f01fa09170b8884d402209e49244#dd2b7b0306d3f01fa09170b8884d402209e49244" +source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1" dependencies = [ "libc", "libnet", @@ -5444,10 +5444,12 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=dd2b7b0306d3f01fa09170b8884d402209e49244#dd2b7b0306d3f01fa09170b8884d402209e49244" +source = "git+https://github.com/oxidecomputer/opte?rev=1d29ef60a18179babfb44f0f7a3c2fe71034a2c1#1d29ef60a18179babfb44f0f7a3c2fe71034a2c1" dependencies = [ + "cfg-if", "illumos-sys-hdrs", "opte", + "poptrie", "serde", "smoltcp 0.11.0", "zerocopy 0.7.31", @@ -6072,6 +6074,11 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "poptrie" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/poptrie?branch=multipath#ca52bef3f87ff1a67d81b3c6e601dcb5fdbcc165" + [[package]] name = "portable-atomic" version = "1.4.3" diff --git a/Cargo.toml b/Cargo.toml index d97236b632..3f70149b3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -267,7 +267,7 @@ omicron-sled-agent = { path = "sled-agent" } omicron-test-utils = { path = "test-utils" } omicron-zone-package = "0.10.1" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "dd2b7b0306d3f01fa09170b8884d402209e49244", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "1d29ef60a18179babfb44f0f7a3c2fe71034a2c1", features = [ "api", "std" ] } once_cell = "1.19.0" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.0.0" @@ -275,7 +275,7 @@ openapiv3 = "2.0.0" openssl = "0.10" openssl-sys = "0.9" openssl-probe = "0.1.5" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "dd2b7b0306d3f01fa09170b8884d402209e49244" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "1d29ef60a18179babfb44f0f7a3c2fe71034a2c1" } oso = "0.27" owo-colors = "3.5.0" oximeter = { path = "oximeter/oximeter" } diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index 93248c73a1..c32345d1dc 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -20,7 +20,7 @@ pub use inner::types; pub use inner::Error; use either::Either; -use inner::types::Ipv6Prefix; +use inner::types::{Ipv6Prefix, TunnelOrigin}; use inner::Client as InnerClient; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; @@ -108,6 +108,22 @@ impl Client { }); } + pub fn advertise_tunnel_endpoint(&self, endpoint: TunnelOrigin) { + let me = self.clone(); + tokio::spawn(async move { + retry_notify(retry_policy_internal_service_aggressive(), || async { + me.inner.advertise_tunnel_endpoints(&vec![endpoint.clone()]).await?; + Ok(()) + }, |err, duration| { + info!( + me.log, + "Failed to notify ddmd of tunnel endpoint (retry in {duration:?}"; + "err" => %err, + ); + }).await.unwrap(); + }); + } + /// Returns the addresses of connected sleds. /// /// Note: These sleds have not yet been verified. diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 710e783181..d06b6b26e5 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -29,26 +29,6 @@ pub use oxide_vpc::api::DhcpCfg; pub use oxide_vpc::api::Vni; use std::net::IpAddr; -fn default_boundary_services() -> BoundaryServices { - use oxide_vpc::api::Ipv6Addr; - use oxide_vpc::api::MacAddr; - // TODO-completeness: Don't hardcode any of these values. - // - // Boundary Services will be started on several Sidecars during rack - // setup, and those addresses and VNIs will need to be propagated here. - // See https://github.com/oxidecomputer/omicron/issues/1382 - let ip = Ipv6Addr::from([0xfd00, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01]); - - // This MAC address is entirely irrelevant to the functionality of OPTE and - // the Oxide VPC. It's never used to actually forward packets. It only - // represents the "logical" destination of Boundary Services as a - // destination that OPTE as a virtual gateway forwards packets to as its - // next hop. - let mac = MacAddr::from_const([0xa8, 0x40, 0x25, 0xf9, 0x99, 0x99]); - let vni = Vni::new(99_u32).unwrap(); - BoundaryServices { ip, mac, vni } -} - /// Information about the gateway for an OPTE port #[derive(Debug, Clone, Copy)] #[allow(dead_code)] diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 3558ef1c78..c472996598 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -4,7 +4,6 @@ //! Manager for all OPTE ports on a Helios system -use crate::opte::default_boundary_services; use crate::opte::opte_firewall_rules; use crate::opte::params::DeleteVirtualNetworkInterfaceHost; use crate::opte::params::SetVirtualNetworkInterfaceHost; @@ -110,7 +109,6 @@ impl PortManager { let subnet = IpNetwork::from(nic.subnet); let vpc_subnet = IpCidr::from(subnet); let gateway = Gateway::from_subnet(&subnet); - let boundary_services = default_boundary_services(); // Describe the external IP addresses for this port. macro_rules! ip_cfg { @@ -219,7 +217,6 @@ impl PortManager { gateway_mac: MacAddr::from(gateway.mac.into_array()), vni, phys_ip: self.inner.underlay_ip.into(), - boundary_services, }; // Create the xde device. diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index 0d6bb52421..979ec54afd 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -15,6 +15,10 @@ use crate::app::sagas::{ use anyhow::Error; use db::datastore::SwitchPortSettingsCombinedResult; use dpd_client::types::PortId; +use mg_admin_client::types::{ + AddStaticRoute4Request, DeleteStaticRoute4Request, Prefix4, StaticRoute4, + StaticRoute4List, +}; use nexus_db_model::NETWORK_KEY; use nexus_db_queries::db::datastore::UpdatePrecondition; use nexus_db_queries::{authn, db}; @@ -52,6 +56,10 @@ declare_saga_actions! { + spa_ensure_switch_port_settings - spa_undo_ensure_switch_port_settings } + ENSURE_SWITCH_ROUTES -> "ensure_switch_routes" { + + spa_ensure_switch_routes + - spa_undo_ensure_switch_routes + } ENSURE_SWITCH_PORT_UPLINK -> "ensure_switch_port_uplink" { + spa_ensure_switch_port_uplink - spa_undo_ensure_switch_port_uplink @@ -210,6 +218,82 @@ async fn spa_ensure_switch_port_settings( Ok(()) } +async fn spa_ensure_switch_routes( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let settings = sagactx + .lookup::("switch_port_settings")?; + + let mut rq = AddStaticRoute4Request { + routes: StaticRoute4List { list: Vec::new() }, + }; + for r in settings.routes { + let nexthop = match r.gw.ip() { + IpAddr::V4(v4) => v4, + IpAddr::V6(_) => continue, + }; + let prefix = match r.gw.ip() { + IpAddr::V4(v4) => Prefix4 { value: v4, length: r.gw.prefix() }, + IpAddr::V6(_) => continue, + }; + let sr = StaticRoute4 { nexthop, prefix }; + rq.routes.list.push(sr); + } + + let mg_client: Arc = + select_mg_client(&sagactx, &opctx, params.switch_port_id).await?; + + mg_client.inner.static_add_v4_route(&rq).await.map_err(|e| { + ActionError::action_failed(format!("mgd static route add {e}")) + })?; + + Ok(()) +} + +async fn spa_undo_ensure_switch_routes( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let settings = sagactx + .lookup::("switch_port_settings")?; + + let mut rq = DeleteStaticRoute4Request { + routes: StaticRoute4List { list: Vec::new() }, + }; + + for r in settings.routes { + let nexthop = match r.gw.ip() { + IpAddr::V4(v4) => v4, + IpAddr::V6(_) => continue, + }; + let prefix = match r.gw.ip() { + IpAddr::V4(v4) => Prefix4 { value: v4, length: r.gw.prefix() }, + IpAddr::V6(_) => continue, + }; + let sr = StaticRoute4 { nexthop, prefix }; + rq.routes.list.push(sr); + } + + let mg_client: Arc = + select_mg_client(&sagactx, &opctx, params.switch_port_id).await?; + + mg_client.inner.static_remove_v4_route(&rq).await.map_err(|e| { + ActionError::action_failed(format!("mgd static route remove {e}")) + })?; + + Ok(()) +} + async fn spa_undo_ensure_switch_port_settings( sagactx: NexusActionContext, ) -> Result<(), Error> { diff --git a/nexus/src/app/sagas/switch_port_settings_clear.rs b/nexus/src/app/sagas/switch_port_settings_clear.rs index 0d876f8159..ff79de8e8e 100644 --- a/nexus/src/app/sagas/switch_port_settings_clear.rs +++ b/nexus/src/app/sagas/switch_port_settings_clear.rs @@ -15,12 +15,16 @@ use crate::app::sagas::{ }; use anyhow::Error; use dpd_client::types::PortId; -use mg_admin_client::types::DeleteNeighborRequest; +use mg_admin_client::types::{ + AddStaticRoute4Request, DeleteNeighborRequest, DeleteStaticRoute4Request, + Prefix4, StaticRoute4, StaticRoute4List, +}; use nexus_db_model::NETWORK_KEY; use nexus_db_queries::authn; use nexus_db_queries::db::datastore::UpdatePrecondition; use omicron_common::api::external::{self, NameOrId, SwitchLocation}; use serde::{Deserialize, Serialize}; +use std::net::IpAddr; use std::str::FromStr; use std::sync::Arc; use steno::ActionError; @@ -43,6 +47,10 @@ declare_saga_actions! { + spa_clear_switch_port_settings - spa_undo_clear_switch_port_settings } + CLEAR_SWITCH_PORT_ROUTES -> "clear_switch_port_routes" { + + spa_clear_switch_port_routes + - spa_undo_clear_switch_port_routes + } CLEAR_SWITCH_PORT_UPLINK -> "clear_switch_port_uplink" { + spa_clear_switch_port_uplink - spa_undo_clear_switch_port_uplink @@ -351,6 +359,108 @@ async fn spa_undo_clear_switch_port_bgp_settings( .await?) } +async fn spa_clear_switch_port_routes( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = + sagactx.lookup::>("original_switch_port_settings_id")?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = nexus + .switch_port_settings_get(&opctx, &NameOrId::Id(id)) + .await + .map_err(ActionError::action_failed)?; + + let mut rq = DeleteStaticRoute4Request { + routes: StaticRoute4List { list: Vec::new() }, + }; + + for r in settings.routes { + let nexthop = match r.gw.ip() { + IpAddr::V4(v4) => v4, + IpAddr::V6(_) => continue, + }; + let prefix = match r.gw.ip() { + IpAddr::V4(v4) => Prefix4 { value: v4, length: r.gw.prefix() }, + IpAddr::V6(_) => continue, + }; + let sr = StaticRoute4 { nexthop, prefix }; + rq.routes.list.push(sr); + } + + let mg_client: Arc = + select_mg_client(&sagactx, &opctx, params.switch_port_id).await?; + + mg_client.inner.static_remove_v4_route(&rq).await.map_err(|e| { + ActionError::action_failed(format!("mgd static route remove {e}")) + })?; + + Ok(()) +} + +async fn spa_undo_clear_switch_port_routes( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = + sagactx.lookup::>("original_switch_port_settings_id")?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = nexus + .switch_port_settings_get(&opctx, &NameOrId::Id(id)) + .await + .map_err(ActionError::action_failed)?; + + let mut rq = AddStaticRoute4Request { + routes: StaticRoute4List { list: Vec::new() }, + }; + + for r in settings.routes { + let nexthop = match r.gw.ip() { + IpAddr::V4(v4) => v4, + IpAddr::V6(_) => continue, + }; + let prefix = match r.gw.ip() { + IpAddr::V4(v4) => Prefix4 { value: v4, length: r.gw.prefix() }, + IpAddr::V6(_) => continue, + }; + let sr = StaticRoute4 { nexthop, prefix }; + rq.routes.list.push(sr); + } + + let mg_client: Arc = + select_mg_client(&sagactx, &opctx, params.switch_port_id).await?; + + mg_client.inner.static_add_v4_route(&rq).await.map_err(|e| { + ActionError::action_failed(format!("mgd static route remove {e}")) + })?; + + Ok(()) +} + async fn spa_clear_switch_port_bootstore_network_settings( sagactx: NexusActionContext, ) -> Result<(), ActionError> { diff --git a/nexus/src/app/sagas/switch_port_settings_common.rs b/nexus/src/app/sagas/switch_port_settings_common.rs index 9ef23ebf44..9c710d837d 100644 --- a/nexus/src/app/sagas/switch_port_settings_common.rs +++ b/nexus/src/app/sagas/switch_port_settings_common.rs @@ -1,12 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + use super::NexusActionContext; use crate::app::map_switch_zone_addrs; use crate::Nexus; use db::datastore::SwitchPortSettingsCombinedResult; use dpd_client::types::{ LinkCreate, LinkId, LinkSettings, PortFec, PortSettings, PortSpeed, - RouteSettingsV4, RouteSettingsV6, }; -use dpd_client::{Ipv4Cidr, Ipv6Cidr}; use internal_dns::ServiceName; use ipnetwork::IpNetwork; use mg_admin_client::types::Prefix4; @@ -85,41 +87,6 @@ pub(crate) fn api_to_dpd_port_settings( ); } - for r in &settings.routes { - match &r.dst { - IpNetwork::V4(n) => { - let gw = match r.gw.ip() { - IpAddr::V4(gw) => gw, - IpAddr::V6(_) => { - return Err( - "IPv4 destination cannot have IPv6 nexthop".into() - ) - } - }; - dpd_port_settings.v4_routes.insert( - Ipv4Cidr { prefix: n.ip(), prefix_len: n.prefix() } - .to_string(), - vec![RouteSettingsV4 { link_id: link_id.0, nexthop: gw }], - ); - } - IpNetwork::V6(n) => { - let gw = match r.gw.ip() { - IpAddr::V6(gw) => gw, - IpAddr::V4(_) => { - return Err( - "IPv6 destination cannot have IPv4 nexthop".into() - ) - } - }; - dpd_port_settings.v6_routes.insert( - Ipv6Cidr { prefix: n.ip(), prefix_len: n.prefix() } - .to_string(), - vec![RouteSettingsV6 { link_id: link_id.0, nexthop: gw }], - ); - } - } - } - Ok(dpd_port_settings) } diff --git a/package-manifest.toml b/package-manifest.toml index 7b12583437..d27ad79b7a 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -438,10 +438,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "2fd39b75df696961e5ea190c7d74dd91f4849cd3" +source.commit = "869cf802efcbd2f0edbb614ed92caa3e3164c1fc" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt -source.sha256 = "38851c79c85d53e997db748520fb27c82299ce7e58a550e35646a548498f1271" +source.sha256 = "1cf9cb514d11275d93c4e4760500539a778f23039374508ca07528fcaf0ba3f8" output.type = "tarball" [package.mg-ddm] @@ -454,10 +454,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "2fd39b75df696961e5ea190c7d74dd91f4849cd3" +source.commit = "869cf802efcbd2f0edbb614ed92caa3e3164c1fc" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "8cd94e9a6f6175081ce78f0281085a08a5306cde453d8e21deb28050945b1d88" +source.sha256 = "a9b959b4287ac2ec7b45ed99ccd00e1f134b8e3d501099cd669cee5de9525ae3" output.type = "zone" output.intermediate_only = true @@ -469,10 +469,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "2fd39b75df696961e5ea190c7d74dd91f4849cd3" +source.commit = "869cf802efcbd2f0edbb614ed92caa3e3164c1fc" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "802636775fa77dc6eec193e65fde87e403f6a11531745d47ef5e7ff13b242890" +source.sha256 = "ab882fbeab54987645492872e67f3351f8d14629a041465cc845ac8583a7002b" output.type = "zone" output.intermediate_only = true diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 75958a2f37..3d4eaaff1a 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -6,21 +6,24 @@ use anyhow::{anyhow, Context}; use bootstore::schemes::v0 as bootstore; -use ddm_admin_client::{Client as DdmAdminClient, DdmError}; -use dpd_client::types::{Ipv6Entry, RouteSettingsV6}; +use ddm_admin_client::DdmError; +use dpd_client::types::Ipv6Entry; use dpd_client::types::{ - LinkCreate, LinkId, LinkSettings, PortId, PortSettings, RouteSettingsV4, + LinkCreate, LinkId, LinkSettings, PortId, PortSettings, }; use dpd_client::Client as DpdClient; use futures::future; use gateway_client::Client as MgsClient; use internal_dns::resolver::{ResolveError, Resolver as DnsResolver}; use internal_dns::ServiceName; -use ipnetwork::{IpNetwork, Ipv6Network}; -use mg_admin_client::types::{ApplyRequest, BgpPeerConfig, Prefix4}; +use ipnetwork::Ipv6Network; +use mg_admin_client::types::{ + AddStaticRoute4Request, ApplyRequest, BgpPeerConfig, Prefix4, StaticRoute4, + StaticRoute4List, +}; use mg_admin_client::Client as MgdClient; -use omicron_common::address::{Ipv6Subnet, MGD_PORT, MGS_PORT}; -use omicron_common::address::{DDMD_PORT, DENDRITE_PORT}; +use omicron_common::address::DENDRITE_PORT; +use omicron_common::address::{MGD_PORT, MGS_PORT}; use omicron_common::api::internal::shared::{ BgpConfig, PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, RackNetworkConfigV1, SwitchLocation, UplinkConfig, @@ -453,13 +456,6 @@ impl<'a> EarlyNetworkSetup<'a> { "unable to apply uplink port configuration: {e}" )) })?; - - info!(self.log, "advertising boundary services loopback address"); - - let ddmd_addr = - SocketAddrV6::new(switch_zone_underlay_ip, DDMD_PORT, 0, 0); - let ddmd_client = DdmAdminClient::new(&self.log, ddmd_addr)?; - ddmd_client.advertise_prefix(Ipv6Subnet::new(ipv6_entry.addr)); } let mgd = MgdClient::new( @@ -548,6 +544,32 @@ impl<'a> EarlyNetworkSetup<'a> { } } + // Iterate through ports and apply static routing config. + let mut rq = AddStaticRoute4Request { + routes: StaticRoute4List { list: Vec::new() }, + }; + for port in &our_ports { + for r in &port.routes { + let nexthop = match r.nexthop { + IpAddr::V4(v4) => v4, + IpAddr::V6(_) => continue, + }; + let prefix = match r.destination.ip() { + IpAddr::V4(v4) => { + Prefix4 { value: v4, length: r.destination.prefix() } + } + IpAddr::V6(_) => continue, + }; + let sr = StaticRoute4 { nexthop, prefix }; + rq.routes.list.push(sr); + } + } + mgd.inner.static_add_v4_route(&rq).await.map_err(|e| { + EarlyNetworkSetupError::BgpConfigurationError(format!( + "static routing configuration failed: {e}", + )) + })?; + Ok(our_ports) } @@ -600,25 +622,6 @@ impl<'a> EarlyNetworkSetup<'a> { )) })?; - for r in &port_config.routes { - if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) = - (r.destination, r.nexthop) - { - dpd_port_settings.v4_routes.insert( - dst.to_string(), - vec![RouteSettingsV4 { link_id: link_id.0, nexthop }], - ); - } - if let (IpNetwork::V6(dst), IpAddr::V6(nexthop)) = - (r.destination, r.nexthop) - { - dpd_port_settings.v6_routes.insert( - dst.to_string(), - vec![RouteSettingsV6 { link_id: link_id.0, nexthop }], - ); - } - } - Ok((ipv6_entry, dpd_port_settings, port_id)) } diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index fdc81c0f8f..12cb2afd24 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -103,7 +103,7 @@ bgp = [] # Routes associated with this port. routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] # Addresses associated with this port. -addresses = ["192.168.1.30/32"] +addresses = ["192.168.1.30/24"] # Name of the uplink port. This should always be "qsfp0" when using softnpu. port = "qsfp0" # The speed of this port. diff --git a/tools/ci_check_opte_ver.sh b/tools/ci_check_opte_ver.sh index 26382690e1..7f05ec1f36 100755 --- a/tools/ci_check_opte_ver.sh +++ b/tools/ci_check_opte_ver.sh @@ -1,6 +1,11 @@ #!/bin/bash set -euo pipefail +source tools/opte_version_override +if [[ "x$OPTE_COMMIT" != "x" ]]; then + exit 0 +fi + # Grab all the oxidecomputer/opte dependencies' revisions readarray -t opte_deps_revs < <(toml get Cargo.toml workspace.dependencies | jq -r 'to_entries | .[] | select(.value.git? | contains("oxidecomputer/opte")?) | .value.rev') OPTE_REV="${opte_deps_revs[0]}" diff --git a/tools/install_opte.sh b/tools/install_opte.sh index 20a33b05a5..b572c305a7 100755 --- a/tools/install_opte.sh +++ b/tools/install_opte.sh @@ -97,3 +97,13 @@ if [[ "$RC" -ne 0 ]]; then echo "The \`opteadm\` administration tool is not on your path." echo "You may add \"/opt/oxide/opte/bin\" to your path to access it." fi + +source $OMICRON_TOP/tools/opte_version_override + +if [[ "x$OPTE_COMMIT" != "x" ]]; then + set +x + curl -fOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde + pfexec rem_drv xde || true + pfexec mv xde /kernel/drv/amd64/xde + pfexec add_drv xde || true +fi diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 37c099d7f5..be8772b7e6 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="2fd39b75df696961e5ea190c7d74dd91f4849cd3" -SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf" +COMMIT="869cf802efcbd2f0edbb614ed92caa3e3164c1fc" +SHA2="0b0dbc2f8bbc5d2d9be92d64c4865f8f9335355aae62f7de9f67f81dfb3f1803" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 329c05fc42..6bf1999c61 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="2fd39b75df696961e5ea190c7d74dd91f4849cd3" -SHA2="931efa310d972b1f8afba2308751fc6a2035afbaebba77b3a40a8358c123ba3c" +COMMIT="869cf802efcbd2f0edbb614ed92caa3e3164c1fc" +SHA2="7618511f905d26394ef7c552339dd78835ce36a6def0d85b05b6d1e363a5e7b4" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 1d3cf98f94..b5fe84b662 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="802636775fa77dc6eec193e65fde87e403f6a11531745d47ef5e7ff13b242890" -MGD_LINUX_SHA256="1bcadfd700902e3640843e0bb53d3defdbcd8d86c3279efa0953ae8d6437e2b0" \ No newline at end of file +CIDL_SHA256="ab882fbeab54987645492872e67f3351f8d14629a041465cc845ac8583a7002b" +MGD_LINUX_SHA256="93331c1001e3aa506a8c1b83346abba1995e489910bff2c94a86730b96617a34" \ No newline at end of file diff --git a/tools/opte_version b/tools/opte_version index 82d79dcf28..0a04873e11 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.27.214 +0.28.215 diff --git a/tools/opte_version_override b/tools/opte_version_override new file mode 100644 index 0000000000..80a6529b24 --- /dev/null +++ b/tools/opte_version_override @@ -0,0 +1,5 @@ +#!/bin/bash + +# only set this if you want to override the version of opte/xde installed by the +# install_opte.sh script +OPTE_COMMIT=""