updates for tunnel routing

.github/buildomat/jobs/deploy.sh

@@ -83,6 +83,13 @@ z_swadm () {
 	pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm $@
 }
 
+# XXX remove. This is just to test against a development branch of OPTE in CI.
+OPTE_COMMIT="373032f452aee26e4c3b429fe9635bea9576f8a8"
+curl  -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde
+pfexec rem_drv xde
+pfexec mv xde /kernel/drv/amd64/xde
+pfexec add_drv xde
+
 #
 # XXX work around 14537 (UFS should not allow directories to be unlinked) which
 # is probably not yet fixed in xde branch?  Once the xde branch merges from

Cargo.toml

@@ -237,15 +237,15 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.8.3"
 oxide-client = { path = "oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "98d33125413f01722947e322f82caf9d22209434", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "373032f452aee26e4c3b429fe9635bea9576f8a8", features = [ "api", "std" ] }
 once_cell = "1.18.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "1.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
 openssl-probe = "0.1.2"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "98d33125413f01722947e322f82caf9d22209434" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "373032f452aee26e4c3b429fe9635bea9576f8a8" }
 oso = "0.26"
 owo-colors = "3.5.0"
 oximeter = { path = "oximeter/oximeter" }

ddm-admin-client/src/lib.rs

@@ -20,7 +20,7 @@ pub use inner::types;
 pub use inner::Error;
 
 use either::Either;
-use inner::types::Ipv6Prefix;
+use inner::types::{Ipv6Prefix, TunnelOrigin};
 use inner::Client as InnerClient;
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::SLED_PREFIX;
@@ -108,6 +108,22 @@ impl Client {
         });
     }
 
+    pub fn advertise_tunnel_endpoint(&self, endpoint: TunnelOrigin) {
+        let me = self.clone();
+        tokio::spawn(async move {
+            retry_notify(retry_policy_internal_service_aggressive(), || async {
+                me.inner.advertise_tunnel_endpoints(&vec![endpoint.clone()]).await?;
+                Ok(())
+            }, |err, duration| {
+                info!(
+                    me.log,
+                    "Failed to notify ddmd of tunnel endpoint (retry in {duration:?}";
+                    "err" => %err,
+                );
+            }).await.unwrap();
+        });
+    }
+
     /// Returns the addresses of connected sleds.
     ///
     /// Note: These sleds have not yet been verified.

illumos-utils/src/opte/mod.rs

@@ -28,26 +28,6 @@ pub use oxide_vpc::api::BoundaryServices;
 pub use oxide_vpc::api::Vni;
 use std::net::IpAddr;
 
-fn default_boundary_services() -> BoundaryServices {
-    use oxide_vpc::api::Ipv6Addr;
-    use oxide_vpc::api::MacAddr;
-    // TODO-completeness: Don't hardcode any of these values.
-    //
-    // Boundary Services will be started on several Sidecars during rack
-    // setup, and those addresses and VNIs will need to be propagated here.
-    // See https://github.com/oxidecomputer/omicron/issues/1382
-    let ip = Ipv6Addr::from([0xfd00, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01]);
-
-    // This MAC address is entirely irrelevant to the functionality of OPTE and
-    // the Oxide VPC. It's never used to actually forward packets. It only
-    // represents the "logical" destination of Boundary Services as a
-    // destination that OPTE as a virtual gateway forwards packets to as its
-    // next hop.
-    let mac = MacAddr::from_const([0xa8, 0x40, 0x25, 0xf9, 0x99, 0x99]);
-    let vni = Vni::new(99_u32).unwrap();
-    BoundaryServices { ip, mac, vni }
-}
-
 /// Information about the gateway for an OPTE port
 #[derive(Debug, Clone, Copy)]
 #[allow(dead_code)]

illumos-utils/src/opte/port_manager.rs

@@ -4,7 +4,6 @@
 
 //! Manager for all OPTE ports on a Helios system
 
-use crate::opte::default_boundary_services;
 use crate::opte::opte_firewall_rules;
 use crate::opte::params::SetVirtualNetworkInterfaceHost;
 use crate::opte::params::VpcFirewallRule;
@@ -105,7 +104,6 @@ impl PortManager {
         let subnet = IpNetwork::from(nic.subnet);
         let vpc_subnet = IpCidr::from(subnet);
         let gateway = Gateway::from_subnet(&subnet);
-        let boundary_services = default_boundary_services();
 
         // Describe the external IP addresses for this port.
         //
@@ -203,7 +201,6 @@ impl PortManager {
             gateway_mac: MacAddr::from(gateway.mac.into_array()),
             vni,
             phys_ip: self.inner.underlay_ip.into(),
-            boundary_services,
             // TODO-completeness (#2153): Plumb domain search list
             domain_list: vec![],
         };

package-manifest.toml

@@ -385,10 +385,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "12703675393459e74139f8140e0b3c4c4f129d5d"
+source.commit = "4c720d08a43c03c95c08e0de2a9b2633053428d4"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/maghemite.sha256.txt
-source.sha256 = "e57fe791ee898d59890c5779fbd4dce598250fb6ed53832024212bcdeec0cc5b"
+source.sha256 = "4ff0b52007846543d3cc5b5d92cbd419ddee4997f9cc1b503ca2b7d3628001e7"
 output.type = "tarball"
 
 [package.mg-ddm]
@@ -401,10 +401,10 @@ source.repo = "maghemite"
 # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when
 # building `ddm-admin-client` (which will instruct you to update
 # `tools/maghemite_openapi_version`).
-source.commit = "12703675393459e74139f8140e0b3c4c4f129d5d"
+source.commit = "4c720d08a43c03c95c08e0de2a9b2633053428d4"
 # The SHA256 digest is automatically posted to:
 # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image/<commit>/mg-ddm.sha256.txt
-source.sha256 = "3aa0d32b1d2b6be7091b9c665657296e924a86a00ca38756e9f45a1e629fd92b"
+source.sha256 = "030d7f379941e4ff02ebb42a45a597fd9842b08e3baa92b9972b432a9fd8b10a"
 output.type = "zone"
 output.intermediate_only = true
 

sled-agent/src/bootstrap/early_networking.rs

@@ -6,6 +6,7 @@
 
 use anyhow::{anyhow, Context};
 use bootstore::schemes::v0 as bootstore;
+use ddm_admin_client::types::{IpPrefix, Ipv4Prefix, TunnelOrigin};
 use ddm_admin_client::{Client as DdmAdminClient, DdmError};
 use dpd_client::types::Ipv6Entry;
 use dpd_client::types::{
@@ -29,7 +30,7 @@ use omicron_common::backoff::{
 use serde::{Deserialize, Serialize};
 use slog::Logger;
 use std::collections::{HashMap, HashSet};
-use std::net::{IpAddr, Ipv6Addr, SocketAddrV6};
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6};
 use std::time::{Duration, Instant};
 use thiserror::Error;
 
@@ -443,7 +444,17 @@ impl<'a> EarlyNetworkSetup<'a> {
             let ddmd_addr =
                 SocketAddrV6::new(switch_zone_underlay_ip, DDMD_PORT, 0, 0);
             let ddmd_client = DdmAdminClient::new(&self.log, ddmd_addr)?;
+
             ddmd_client.advertise_prefix(Ipv6Subnet::new(ipv6_entry.addr));
+
+            ddmd_client.advertise_tunnel_endpoint(TunnelOrigin {
+                overlay_prefix: IpPrefix::V4(Ipv4Prefix {
+                    addr: Ipv4Addr::UNSPECIFIED,
+                    len: 0,
+                }),
+                boundary_addr: ipv6_entry.addr,
+                vni: 99,
+            });
         }
 
         Ok(our_uplinks)

tools/install_opte.sh

@@ -70,3 +70,12 @@ if [[ "$RC" -ne 0 ]]; then
     echo "The \`opteadm\` administration tool is not on your path."
     echo "You may add \"/opt/oxide/opte/bin\" to your path to access it."
 fi
+
+source $OMICRON_TOP/tools/opte_version_override
+
+if [[ "x$OPTE_COMMIT" != "x" ]]; then
+    curl -fOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde
+    pfexec rem_drv xde
+    pfexec mv xde /kernel/drv/amd64/xde
+    pfexec add_drv xde
+fi

tools/maghemite_openapi_version

@@ -1,2 +1,2 @@
-COMMIT="12703675393459e74139f8140e0b3c4c4f129d5d"
-SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf"
+COMMIT="4c720d08a43c03c95c08e0de2a9b2633053428d4"
+SHA2="542dd084fe2abe548236dbc3921d6bd3fa8b62d1a5b727904b954f6db6226462"

tools/opte_version_override

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# only set this if you want to override the version of opte/xde installed by the
+# install_opte.sh script
+OPTE_COMMIT="373032f452aee26e4c3b429fe9635bea9576f8a8"