updates for tunnel routing

.github/buildomat/jobs/deploy.sh

@@ -97,6 +97,15 @@ z_swadm () {
 	pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm $@
 }
 
+# XXX remove. This is just to test against a development branch of OPTE in CI.
+set +x
+OPTE_COMMIT="82ea7ca153442194657cd575693d2a8fcdd68a96"
+curl  -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde
+pfexec rem_drv xde || true
+pfexec mv xde /kernel/drv/amd64/xde
+pfexec add_drv xde || true
+set -x
+
 #
 # XXX work around 14537 (UFS should not allow directories to be unlinked) which
 # is probably not yet fixed in xde branch?  Once the xde branch merges from

Cargo.toml

@@ -260,15 +260,15 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.9.1"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "4e6e6ab6379fa4bc40f5d0c7340b9f35c45ad6e4", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "82ea7ca153442194657cd575693d2a8fcdd68a96", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
 openssl-probe = "0.1.5"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "4e6e6ab6379fa4bc40f5d0c7340b9f35c45ad6e4" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "82ea7ca153442194657cd575693d2a8fcdd68a96" }
 oso = "0.27"
 owo-colors = "3.5.0"
 oximeter = { path = "oximeter/oximeter" }

clients/ddm-admin-client/src/lib.rs

@@ -20,7 +20,7 @@ pub use inner::types;
 pub use inner::Error;
 
 use either::Either;
-use inner::types::Ipv6Prefix;
+use inner::types::{Ipv6Prefix, TunnelOrigin};
 use inner::Client as InnerClient;
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::SLED_PREFIX;
@@ -108,6 +108,22 @@ impl Client {
         });
     }
 
+    pub fn advertise_tunnel_endpoint(&self, endpoint: TunnelOrigin) {
+        let me = self.clone();
+        tokio::spawn(async move {
+            retry_notify(retry_policy_internal_service_aggressive(), || async {
+                me.inner.advertise_tunnel_endpoints(&vec![endpoint.clone()]).await?;
+                Ok(())
+            }, |err, duration| {
+                info!(
+                    me.log,
+                    "Failed to notify ddmd of tunnel endpoint (retry in {duration:?}";
+                    "err" => %err,
+                );
+            }).await.unwrap();
+        });
+    }
+
     /// Returns the addresses of connected sleds.
     ///
     /// Note: These sleds have not yet been verified.

illumos-utils/src/opte/mod.rs

@@ -29,26 +29,6 @@ pub use oxide_vpc::api::DhcpCfg;
 pub use oxide_vpc::api::Vni;
 use std::net::IpAddr;
 
-fn default_boundary_services() -> BoundaryServices {
-    use oxide_vpc::api::Ipv6Addr;
-    use oxide_vpc::api::MacAddr;
-    // TODO-completeness: Don't hardcode any of these values.
-    //
-    // Boundary Services will be started on several Sidecars during rack
-    // setup, and those addresses and VNIs will need to be propagated here.
-    // See https://github.com/oxidecomputer/omicron/issues/1382
-    let ip = Ipv6Addr::from([0xfd00, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01]);
-
-    // This MAC address is entirely irrelevant to the functionality of OPTE and
-    // the Oxide VPC. It's never used to actually forward packets. It only
-    // represents the "logical" destination of Boundary Services as a
-    // destination that OPTE as a virtual gateway forwards packets to as its
-    // next hop.
-    let mac = MacAddr::from_const([0xa8, 0x40, 0x25, 0xf9, 0x99, 0x99]);
-    let vni = Vni::new(99_u32).unwrap();
-    BoundaryServices { ip, mac, vni }
-}
-
 /// Information about the gateway for an OPTE port
 #[derive(Debug, Clone, Copy)]
 #[allow(dead_code)]

illumos-utils/src/opte/port_manager.rs

@@ -4,7 +4,6 @@
 
 //! Manager for all OPTE ports on a Helios system
 
-use crate::opte::default_boundary_services;
 use crate::opte::opte_firewall_rules;
 use crate::opte::params::DeleteVirtualNetworkInterfaceHost;
 use crate::opte::params::SetVirtualNetworkInterfaceHost;
@@ -110,7 +109,6 @@ impl PortManager {
         let subnet = IpNetwork::from(nic.subnet);
         let vpc_subnet = IpCidr::from(subnet);
         let gateway = Gateway::from_subnet(&subnet);
-        let boundary_services = default_boundary_services();
 
         // Describe the external IP addresses for this port.
         macro_rules! ip_cfg {
@@ -219,7 +217,6 @@ impl PortManager {
             gateway_mac: MacAddr::from(gateway.mac.into_array()),
             vni,
             phys_ip: self.inner.underlay_ip.into(),
-            boundary_services,
         };
 
         // Create the xde device.

package-manifest.toml

@@ -437,10 +437,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "2fd39b75df696961e5ea190c7d74dd91f4849cd3"
+source.commit = "8f52bcb2086a350e0b68d78a3d2ddf543725c434"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/maghemite.sha256.txt
-source.sha256 = "38851c79c85d53e997db748520fb27c82299ce7e58a550e35646a548498f1271"
+source.sha256 = "ddf6221f3c52d1e004252f20ee6b7ddebb336ef4366507a5c04df15ba374abd3"
 output.type = "tarball"
 
 [package.mg-ddm]
@@ -453,10 +453,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "2fd39b75df696961e5ea190c7d74dd91f4849cd3"
+source.commit = "8f52bcb2086a350e0b68d78a3d2ddf543725c434"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "8cd94e9a6f6175081ce78f0281085a08a5306cde453d8e21deb28050945b1d88"
+source.sha256 = "60dd991acfa9652fb53008a403e37556f1ade92954c4870ea037f15da2c9a655"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -468,10 +468,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "2fd39b75df696961e5ea190c7d74dd91f4849cd3"
+source.commit = "8f52bcb2086a350e0b68d78a3d2ddf543725c434"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "802636775fa77dc6eec193e65fde87e403f6a11531745d47ef5e7ff13b242890"
+source.sha256 = "289217a149336ec6807566f0726f95e29609d0537710bf769d5277c26483117f"
 output.type = "zone"
 output.intermediate_only = true
 

sled-agent/src/bootstrap/early_networking.rs

@@ -6,6 +6,7 @@
 
 use anyhow::{anyhow, Context};
 use bootstore::schemes::v0 as bootstore;
+use ddm_admin_client::types::{IpPrefix, Ipv4Prefix, TunnelOrigin};
 use ddm_admin_client::{Client as DdmAdminClient, DdmError};
 use dpd_client::types::{Ipv6Entry, RouteSettingsV6};
 use dpd_client::types::{
@@ -459,7 +460,17 @@ impl<'a> EarlyNetworkSetup<'a> {
             let ddmd_addr =
                 SocketAddrV6::new(switch_zone_underlay_ip, DDMD_PORT, 0, 0);
             let ddmd_client = DdmAdminClient::new(&self.log, ddmd_addr)?;
+
             ddmd_client.advertise_prefix(Ipv6Subnet::new(ipv6_entry.addr));
+
+            ddmd_client.advertise_tunnel_endpoint(TunnelOrigin {
+                overlay_prefix: IpPrefix::V4(Ipv4Prefix {
+                    addr: Ipv4Addr::UNSPECIFIED,
+                    len: 0,
+                }),
+                boundary_addr: ipv6_entry.addr,
+                vni: 99,
+            });
         }
 
         let mgd = MgdClient::new(

tools/ci_check_opte_ver.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
 set -euo pipefail
 
+source tools/opte_version_override
+if [[ "x$OPTE_COMMIT" != "x" ]]; then
+    exit 0
+fi
+
 # Grab all the oxidecomputer/opte dependencies' revisions
 readarray -t opte_deps_revs < <(toml get Cargo.toml workspace.dependencies | jq -r 'to_entries | .[] | select(.value.git? | contains("oxidecomputer/opte")?) | .value.rev')
 OPTE_REV="${opte_deps_revs[0]}"

tools/install_opte.sh

@@ -97,3 +97,13 @@ if [[ "$RC" -ne 0 ]]; then
     echo "The \`opteadm\` administration tool is not on your path."
     echo "You may add \"/opt/oxide/opte/bin\" to your path to access it."
 fi
+
+source $OMICRON_TOP/tools/opte_version_override
+
+if [[ "x$OPTE_COMMIT" != "x" ]]; then
+    set +x
+    curl -fOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde
+    pfexec rem_drv xde || true
+    pfexec mv xde /kernel/drv/amd64/xde
+    pfexec add_drv xde || true
+fi

tools/maghemite_ddm_openapi_version

@@ -1,2 +1,2 @@
-COMMIT="2fd39b75df696961e5ea190c7d74dd91f4849cd3"
-SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf"
+COMMIT="8f52bcb2086a350e0b68d78a3d2ddf543725c434"
+SHA2="0b0dbc2f8bbc5d2d9be92d64c4865f8f9335355aae62f7de9f67f81dfb3f1803"

tools/maghemite_mg_openapi_version

@@ -1,2 +1,2 @@
-COMMIT="2fd39b75df696961e5ea190c7d74dd91f4849cd3"
+COMMIT="8f52bcb2086a350e0b68d78a3d2ddf543725c434"
 SHA2="931efa310d972b1f8afba2308751fc6a2035afbaebba77b3a40a8358c123ba3c"

tools/maghemite_mgd_checksums

@@ -1,2 +1,2 @@
-CIDL_SHA256="802636775fa77dc6eec193e65fde87e403f6a11531745d47ef5e7ff13b242890"
-MGD_LINUX_SHA256="1bcadfd700902e3640843e0bb53d3defdbcd8d86c3279efa0953ae8d6437e2b0"
+CIDL_SHA256="289217a149336ec6807566f0726f95e29609d0537710bf769d5277c26483117f"
+MGD_LINUX_SHA256="4a9bcaa1eb8d84627bf18c653448ad9a84aab098a27288b56a1fcab04d891754"

tools/opte_version

@@ -1 +1 @@
-0.27.202
+0.28.211

tools/opte_version_override

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# only set this if you want to override the version of opte/xde installed by the
+# install_opte.sh script
+OPTE_COMMIT="fe70f180be97553d5e41e3f5272c74ff252a5190"