VPC Subnet Routing [1/2] -- RPW and System Routers

.github/buildomat/jobs/a4x2-deploy.sh

@@ -2,7 +2,7 @@
 #:
 #: name = "a4x2-deploy"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.29"
+#: target = "lab-2.0-opte-0.31"
 #: output_rules = [
 #:	"/out/falcon/*.log",
 #:	"/out/falcon/*.err",

.github/buildomat/jobs/deploy.sh

@@ -2,7 +2,7 @@
 #:
 #: name = "helios / deploy"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.29"
+#: target = "lab-2.0-opte-0.31"
 #: output_rules = [
 #:  "%/var/svc/log/oxide-sled-agent:default.log*",
 #:  "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",
@@ -50,6 +50,7 @@ _exit_trap() {
 		dump-state
 	pfexec /opt/oxide/opte/bin/opteadm list-ports
 	pfexec /opt/oxide/opte/bin/opteadm dump-v2b
+	pfexec /opt/oxide/opte/bin/opteadm dump-v2p
 	z_swadm link ls
 	z_swadm addr list
 	z_swadm route list

Cargo.toml

@@ -312,8 +312,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] }
 maplit = "1.0.2"
 mockall = "0.12"
 newtype_derive = "0.1.6"
-mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" }
-ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "025389ff39d594bf2b815377e2c1dc4dd23b1f96" }
+mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" }
+ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "a557af774b2c3cfe1e9e4e27de9ad85de6a02a98" }
 multimap = "0.10.0"
 nexus-client = { path = "clients/nexus-client" }
 nexus-config = { path = "nexus-config" }
@@ -347,14 +347,14 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "194a8d1d6443f78d59702a25849607dba33db732", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "a0294dbf886bcaf76d33b5c8f0ca6616a9b90781", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "194a8d1d6443f78d59702a25849607dba33db732" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "a0294dbf886bcaf76d33b5c8f0ca6616a9b90781" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }

clients/sled-agent-client/src/lib.rs

@@ -35,6 +35,7 @@ progenitor::generate_api!(
         PortConfigV1 = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
         RouteConfig = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
         IpNet = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
+        RouterId = { derives = [PartialEq, Eq, Hash, Debug, Deserialize, Serialize] },
         VirtualNetworkInterfaceHost = { derives = [PartialEq, Eq, Hash, Serialize, Deserialize] },
         OmicronPhysicalDiskConfig = { derives = [Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord] },
     },
@@ -52,6 +53,11 @@ progenitor::generate_api!(
         IpNetwork = ipnetwork::IpNetwork,
         PortFec = omicron_common::api::internal::shared::PortFec,
         PortSpeed = omicron_common::api::internal::shared::PortSpeed,
+        RouterId = omicron_common::api::internal::shared::RouterId,
+        ReifiedVpcRoute = omicron_common::api::internal::shared::ReifiedVpcRoute,
+        ReifiedVpcRouteSet = omicron_common::api::internal::shared::ReifiedVpcRouteSet,
+        RouterTarget = omicron_common::api::internal::shared::RouterTarget,
+        RouterVersion = omicron_common::api::internal::shared::RouterVersion,
         SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig,
         Vni = omicron_common::api::external::Vni,
         NetworkInterface = omicron_common::api::internal::shared::NetworkInterface,

common/src/api/external/mod.rs

@@ -1680,6 +1680,9 @@ pub enum RouteTarget {
     #[display("inetgw:{0}")]
     /// Forward traffic to an internet gateway
     InternetGateway(Name),
+    #[display("drop")]
+    /// Drop matching traffic
+    Drop,
 }
 
 /// A `RouteDestination` is used to match traffic with a routing rule, on the

common/src/api/internal/shared.rs

@@ -6,13 +6,13 @@
 
 use crate::{
     address::NUM_SOURCE_NAT_PORTS,
-    api::external::{self, BfdMode, ImportExportPolicy, IpNet, Name},
+    api::external::{self, BfdMode, ImportExportPolicy, IpNet, Name, Vni},
 };
 use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     fmt,
     net::{IpAddr, Ipv4Addr, Ipv6Addr},
     str::FromStr,
@@ -590,6 +590,67 @@ impl TryFrom<&[IpNetwork]> for IpAllowList {
     }
 }
 
+/// A VPC route resolved into a concrete target.
+#[derive(
+    Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct ReifiedVpcRoute {
+    pub dest: IpNet,
+    pub target: RouterTarget,
+}
+
+/// The target for a given router entry.
+#[derive(
+    Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+#[serde(tag = "type", rename_all = "snake_case", content = "value")]
+pub enum RouterTarget {
+    Drop,
+    InternetGateway,
+    Ip(IpAddr),
+    VpcSubnet(IpNet),
+}
+
+/// XXX
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct RouterVersion {
+    pub router_id: Uuid,
+    pub generation: u64,
+}
+
+impl RouterVersion {
+    pub fn is_replaced_by(&self, other: &Self) -> bool {
+        (self.router_id != other.router_id)
+            || self.generation < other.generation
+    }
+}
+
+/// Implementation details on XXX
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct RouterId {
+    pub vni: Vni,
+    pub subnet: Option<IpNet>,
+}
+
+/// Version information
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
+pub struct ReifiedVpcRouteState {
+    pub id: RouterId,
+    pub version: Option<RouterVersion>,
+}
+
+/// An updated set of routes for a given
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
+pub struct ReifiedVpcRouteSet {
+    pub id: RouterId,
+    pub version: Option<RouterVersion>,
+    pub routes: HashSet<ReifiedVpcRoute>,
+}
+
 #[cfg(test)]
 mod tests {
     use crate::api::{

dev-tools/omdb/tests/env.out

@@ -118,6 +118,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT
@@ -233,6 +237,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.
@@ -335,6 +343,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.

dev-tools/omdb/tests/successes.out

@@ -295,6 +295,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -482,6 +486,13 @@ task: "v2p_manager"
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
 warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {})
 
+task: "vpc_route_manager"
+  configured period: every 30s
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+warning: unknown background task: "vpc_route_manager" (don't know how to interpret details: Object {})
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/

dev-tools/xtask/src/virtual_hardware.rs

@@ -104,6 +104,7 @@ const IPADM: &'static str = "/usr/sbin/ipadm";
 const MODINFO: &'static str = "/usr/sbin/modinfo";
 const MODUNLOAD: &'static str = "/usr/sbin/modunload";
 const NETSTAT: &'static str = "/usr/bin/netstat";
+const OPTEADM: &'static str = "/opt/oxide/opte/bin/opteadm";
 const PFEXEC: &'static str = "/usr/bin/pfexec";
 const PING: &'static str = "/usr/sbin/ping";
 const SWAP: &'static str = "/usr/sbin/swap";
@@ -247,8 +248,17 @@ fn unload_xde_driver() -> Result<()> {
         println!("xde driver already unloaded");
         return Ok(());
     };
-    println!("unloading xde driver");
+    println!("unloading xde driver:\na) clearing underlay...");
+    let mut cmd = Command::new(PFEXEC);
+    cmd.args([OPTEADM, "clear-xde-underlay"]);
+    if let Err(e) = execute(cmd) {
+        // This is explicitly non-fatal: the underlay is only set when
+        // sled-agent is running. We still need to be able to tear
+        // down the driver if we immediately call create->destroy.
+        println!("\tFailed or already unset: {e}");
+    }
 
+    println!("b) unloading module...");
     let mut cmd = Command::new(PFEXEC);
     cmd.arg(MODUNLOAD);
     cmd.arg("-i");

illumos-utils/src/opte/firewall_rules.rs

@@ -4,6 +4,7 @@
 
 //! Convert Omicron VPC firewall rules to OPTE firewall rules.
 
+use super::net_to_cidr;
 use crate::opte::params::VpcFirewallRule;
 use crate::opte::Vni;
 use macaddr::MacAddr6;
@@ -19,11 +20,6 @@ use oxide_vpc::api::Filters;
 use oxide_vpc::api::FirewallAction;
 use oxide_vpc::api::FirewallRule;
 use oxide_vpc::api::IpAddr;
-use oxide_vpc::api::IpCidr;
-use oxide_vpc::api::Ipv4Cidr;
-use oxide_vpc::api::Ipv4PrefixLen;
-use oxide_vpc::api::Ipv6Cidr;
-use oxide_vpc::api::Ipv6PrefixLen;
 use oxide_vpc::api::Ports;
 use oxide_vpc::api::ProtoFilter;
 use oxide_vpc::api::Protocol;
@@ -70,23 +66,12 @@ impl FromVpcFirewallRule for VpcFirewallRule {
                     {
                         Address::Ip(IpAddr::Ip4(net.ip().into()))
                     }
-                    HostIdentifier::Ip(IpNet::V4(net)) => {
-                        Address::Subnet(IpCidr::Ip4(Ipv4Cidr::new(
-                            net.ip().into(),
-                            Ipv4PrefixLen::new(net.prefix()).unwrap(),
-                        )))
-                    }
                     HostIdentifier::Ip(IpNet::V6(net))
                         if net.prefix() == 128 =>
                     {
                         Address::Ip(IpAddr::Ip6(net.ip().into()))
                     }
-                    HostIdentifier::Ip(IpNet::V6(net)) => {
-                        Address::Subnet(IpCidr::Ip6(Ipv6Cidr::new(
-                            net.ip().into(),
-                            Ipv6PrefixLen::new(net.prefix()).unwrap(),
-                        )))
-                    }
+                    HostIdentifier::Ip(ip) => Address::Subnet(net_to_cidr(*ip)),
                     HostIdentifier::Vpc(vni) => {
                         Address::Vni(Vni::new(u32::from(*vni)).unwrap())
                     }