updates for tunnel routing

.github/buildomat/jobs/deploy.sh

@@ -2,7 +2,7 @@
 #:
 #: name = "helios / deploy"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.27"
+#: target = "lab-2.0-opte-0.28"
 #: output_rules = [
 #:  "%/var/svc/log/oxide-sled-agent:default.log*",
 #:  "%/pool/ext/*/crypt/zone/oxz_*/root/var/svc/log/oxide-*.log*",
@@ -33,6 +33,9 @@ _exit_trap() {
 	local status=$?
 	[[ $status -eq 0 ]] && exit 0
 
+    # XXX paranoia
+    pfexec cp /tmp/opteadm /opt/oxide/opte/bin/opteadm
+
 	set +o errexit
 	set -o xtrace
 	banner evidence
@@ -50,6 +53,7 @@ _exit_trap() {
 		standalone \
 		dump-state
 	pfexec /opt/oxide/opte/bin/opteadm list-ports
+	pfexec /opt/oxide/opte/bin/opteadm dump-v2b
 	z_swadm link ls
 	z_swadm addr list
 	z_swadm route list
@@ -97,6 +101,19 @@ z_swadm () {
 	pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm $@
 }
 
+# XXX remove. This is just to test against a development branch of OPTE in CI.
+set +x
+OPTE_COMMIT="73d4669ea213d0b7aca35c4babb6fd09ed51d29e"
+curl  -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde
+pfexec rem_drv xde || true
+pfexec mv xde /kernel/drv/amd64/xde
+pfexec add_drv xde || true
+curl -sSfOL https://buildomat.eng.oxide.computer/wg/0/artefact/01HM09S4M15WNXB2B2MX8R1GBT/yLalJU5vT4S4IEpwSeY4hPuspxw3JcINokZmlfNU14npHkzG/01HM09SJ2RQSFGW7MVKC9JKZ8D/01HM0A58D888AJ7YP6N1Q6T6ZD/opteadm
+chmod +x opteadm
+cp opteadm /tmp/opteadm
+pfexec mv opteadm /opt/oxide/opte/bin/opteadm
+set -x
+
 #
 # XXX work around 14537 (UFS should not allow directories to be unlinked) which
 # is probably not yet fixed in xde branch?  Once the xde branch merges from
@@ -236,7 +253,7 @@ infra_ip_last = \"$UPLINK_IP\"
 		/^routes/c\\
 routes = \\[{nexthop = \"$GATEWAY_IP\", destination = \"0.0.0.0/0\"}\\]
 		/^addresses/c\\
-addresses = \\[\"$UPLINK_IP/32\"\\]
+addresses = \\[\"$UPLINK_IP/24\"\\]
 	}
 " pkg/config-rss.toml
 diff -u pkg/config-rss.toml{~,} || true

Cargo.toml

@@ -267,15 +267,15 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.10.1"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "dd2b7b0306d3f01fa09170b8884d402209e49244", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "1d29ef60a18179babfb44f0f7a3c2fe71034a2c1", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
 openssl-probe = "0.1.5"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "dd2b7b0306d3f01fa09170b8884d402209e49244" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "1d29ef60a18179babfb44f0f7a3c2fe71034a2c1" }
 oso = "0.27"
 owo-colors = "3.5.0"
 oximeter = { path = "oximeter/oximeter" }

clients/ddm-admin-client/src/lib.rs

@@ -20,7 +20,7 @@ pub use inner::types;
 pub use inner::Error;
 
 use either::Either;
-use inner::types::Ipv6Prefix;
+use inner::types::{Ipv6Prefix, TunnelOrigin};
 use inner::Client as InnerClient;
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::SLED_PREFIX;
@@ -108,6 +108,22 @@ impl Client {
         });
     }
 
+    pub fn advertise_tunnel_endpoint(&self, endpoint: TunnelOrigin) {
+        let me = self.clone();
+        tokio::spawn(async move {
+            retry_notify(retry_policy_internal_service_aggressive(), || async {
+                me.inner.advertise_tunnel_endpoints(&vec![endpoint.clone()]).await?;
+                Ok(())
+            }, |err, duration| {
+                info!(
+                    me.log,
+                    "Failed to notify ddmd of tunnel endpoint (retry in {duration:?}";
+                    "err" => %err,
+                );
+            }).await.unwrap();
+        });
+    }
+
     /// Returns the addresses of connected sleds.
     ///
     /// Note: These sleds have not yet been verified.

illumos-utils/src/opte/mod.rs

@@ -29,26 +29,6 @@ pub use oxide_vpc::api::DhcpCfg;
 pub use oxide_vpc::api::Vni;
 use std::net::IpAddr;
 
-fn default_boundary_services() -> BoundaryServices {
-    use oxide_vpc::api::Ipv6Addr;
-    use oxide_vpc::api::MacAddr;
-    // TODO-completeness: Don't hardcode any of these values.
-    //
-    // Boundary Services will be started on several Sidecars during rack
-    // setup, and those addresses and VNIs will need to be propagated here.
-    // See https://github.com/oxidecomputer/omicron/issues/1382
-    let ip = Ipv6Addr::from([0xfd00, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01]);
-
-    // This MAC address is entirely irrelevant to the functionality of OPTE and
-    // the Oxide VPC. It's never used to actually forward packets. It only
-    // represents the "logical" destination of Boundary Services as a
-    // destination that OPTE as a virtual gateway forwards packets to as its
-    // next hop.
-    let mac = MacAddr::from_const([0xa8, 0x40, 0x25, 0xf9, 0x99, 0x99]);
-    let vni = Vni::new(99_u32).unwrap();
-    BoundaryServices { ip, mac, vni }
-}
-
 /// Information about the gateway for an OPTE port
 #[derive(Debug, Clone, Copy)]
 #[allow(dead_code)]

illumos-utils/src/opte/port_manager.rs

@@ -4,7 +4,6 @@
 
 //! Manager for all OPTE ports on a Helios system
 
-use crate::opte::default_boundary_services;
 use crate::opte::opte_firewall_rules;
 use crate::opte::params::DeleteVirtualNetworkInterfaceHost;
 use crate::opte::params::SetVirtualNetworkInterfaceHost;
@@ -110,7 +109,6 @@ impl PortManager {
         let subnet = IpNetwork::from(nic.subnet);
         let vpc_subnet = IpCidr::from(subnet);
         let gateway = Gateway::from_subnet(&subnet);
-        let boundary_services = default_boundary_services();
 
         // Describe the external IP addresses for this port.
         macro_rules! ip_cfg {
@@ -219,7 +217,6 @@ impl PortManager {
             gateway_mac: MacAddr::from(gateway.mac.into_array()),
             vni,
             phys_ip: self.inner.underlay_ip.into(),
-            boundary_services,
         };
 
         // Create the xde device.

nexus/src/app/sagas/switch_port_settings_apply.rs

@@ -15,6 +15,10 @@ use crate::app::sagas::{
 use anyhow::Error;
 use db::datastore::SwitchPortSettingsCombinedResult;
 use dpd_client::types::PortId;
+use mg_admin_client::types::{
+    AddStaticRoute4Request, DeleteStaticRoute4Request, Prefix4, StaticRoute4,
+    StaticRoute4List,
+};
 use nexus_db_model::NETWORK_KEY;
 use nexus_db_queries::db::datastore::UpdatePrecondition;
 use nexus_db_queries::{authn, db};
@@ -52,6 +56,10 @@ declare_saga_actions! {
         + spa_ensure_switch_port_settings
         - spa_undo_ensure_switch_port_settings
     }
+    ENSURE_SWITCH_ROUTES -> "ensure_switch_routes" {
+        + spa_ensure_switch_routes
+        - spa_undo_ensure_switch_routes
+    }
     ENSURE_SWITCH_PORT_UPLINK -> "ensure_switch_port_uplink" {
         + spa_ensure_switch_port_uplink
         - spa_undo_ensure_switch_port_uplink
@@ -210,6 +218,82 @@ async fn spa_ensure_switch_port_settings(
     Ok(())
 }
 
+async fn spa_ensure_switch_routes(
+    sagactx: NexusActionContext,
+) -> Result<(), ActionError> {
+    let params = sagactx.saga_params::<Params>()?;
+    let opctx = crate::context::op_context_for_saga_action(
+        &sagactx,
+        &params.serialized_authn,
+    );
+
+    let settings = sagactx
+        .lookup::<SwitchPortSettingsCombinedResult>("switch_port_settings")?;
+
+    let mut rq = AddStaticRoute4Request {
+        routes: StaticRoute4List { list: Vec::new() },
+    };
+    for r in settings.routes {
+        let nexthop = match r.gw.ip() {
+            IpAddr::V4(v4) => v4,
+            IpAddr::V6(_) => continue,
+        };
+        let prefix = match r.gw.ip() {
+            IpAddr::V4(v4) => Prefix4 { value: v4, length: r.gw.prefix() },
+            IpAddr::V6(_) => continue,
+        };
+        let sr = StaticRoute4 { nexthop, prefix };
+        rq.routes.list.push(sr);
+    }
+
+    let mg_client: Arc<mg_admin_client::Client> =
+        select_mg_client(&sagactx, &opctx, params.switch_port_id).await?;
+
+    mg_client.inner.static_add_v4_route(&rq).await.map_err(|e| {
+        ActionError::action_failed(format!("mgd static route add {e}"))
+    })?;
+
+    Ok(())
+}
+
+async fn spa_undo_ensure_switch_routes(
+    sagactx: NexusActionContext,
+) -> Result<(), Error> {
+    let params = sagactx.saga_params::<Params>()?;
+    let opctx = crate::context::op_context_for_saga_action(
+        &sagactx,
+        &params.serialized_authn,
+    );
+    let settings = sagactx
+        .lookup::<SwitchPortSettingsCombinedResult>("switch_port_settings")?;
+
+    let mut rq = DeleteStaticRoute4Request {
+        routes: StaticRoute4List { list: Vec::new() },
+    };
+
+    for r in settings.routes {
+        let nexthop = match r.gw.ip() {
+            IpAddr::V4(v4) => v4,
+            IpAddr::V6(_) => continue,
+        };
+        let prefix = match r.gw.ip() {
+            IpAddr::V4(v4) => Prefix4 { value: v4, length: r.gw.prefix() },
+            IpAddr::V6(_) => continue,
+        };
+        let sr = StaticRoute4 { nexthop, prefix };
+        rq.routes.list.push(sr);
+    }
+
+    let mg_client: Arc<mg_admin_client::Client> =
+        select_mg_client(&sagactx, &opctx, params.switch_port_id).await?;
+
+    mg_client.inner.static_remove_v4_route(&rq).await.map_err(|e| {
+        ActionError::action_failed(format!("mgd static route remove {e}"))
+    })?;
+
+    Ok(())
+}
+
 async fn spa_undo_ensure_switch_port_settings(
     sagactx: NexusActionContext,
 ) -> Result<(), Error> {