VPC Subnet Routing [1/2] -- RPW and System Routers

.github/buildomat/jobs/deploy.sh

@@ -2,7 +2,7 @@
 #:
 #: name = "helios / deploy"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.31"
+#: target = "lab-2.0-opte-0.32"
 #: output_rules = [
 #:  "%/var/svc/log/oxide-sled-agent:default.log*",
 #:  "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",

Cargo.toml

@@ -328,8 +328,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] }
 maplit = "1.0.2"
 mockall = "0.12"
 newtype_derive = "0.1.6"
-mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5630887d0373857f77cb264f84aa19bdec720ce3" }
-ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5630887d0373857f77cb264f84aa19bdec720ce3" }
+mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" }
+ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "e63f6d408908b3332d7cd89a4dd44a0f980d931d" }
 multimap = "0.10.0"
 nexus-auth = { path = "nexus/auth" }
 nexus-client = { path = "clients/nexus-client" }
@@ -367,14 +367,14 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "417f74e94978c23f3892ac328c3387f3ecd9bb29", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "915975f6d1729db95619f752148974016912412f", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "417f74e94978c23f3892ac328c3387f3ecd9bb29" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "915975f6d1729db95619f752148974016912412f" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }

clients/sled-agent-client/src/lib.rs

@@ -51,6 +51,11 @@ progenitor::generate_api!(
         NetworkInterface = omicron_common::api::internal::shared::NetworkInterface,
         PortFec = omicron_common::api::internal::shared::PortFec,
         PortSpeed = omicron_common::api::internal::shared::PortSpeed,
+        RouterId = omicron_common::api::internal::shared::RouterId,
+        ResolvedVpcRoute = omicron_common::api::internal::shared::ResolvedVpcRoute,
+        ResolvedVpcRouteSet = omicron_common::api::internal::shared::ResolvedVpcRouteSet,
+        RouterTarget = omicron_common::api::internal::shared::RouterTarget,
+        RouterVersion = omicron_common::api::internal::shared::RouterVersion,
         SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig,
         SwitchLocation = omicron_common::api::external::SwitchLocation,
         TypedUuidForInstanceKind = omicron_uuid_kinds::InstanceUuid,

common/src/api/external/mod.rs

@@ -1331,6 +1331,9 @@ pub enum RouteTarget {
     #[display("inetgw:{0}")]
     /// Forward traffic to an internet gateway
     InternetGateway(Name),
+    #[display("drop")]
+    /// Drop matching traffic
+    Drop,
 }
 
 /// A `RouteDestination` is used to match traffic with a routing rule, on the

common/src/api/internal/shared.rs

@@ -6,13 +6,13 @@
 
 use crate::{
     address::NUM_SOURCE_NAT_PORTS,
-    api::external::{self, BfdMode, ImportExportPolicy, Name},
+    api::external::{self, BfdMode, ImportExportPolicy, Name, Vni},
 };
 use oxnet::{IpNet, Ipv4Net, Ipv6Net};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     fmt,
     net::{IpAddr, Ipv4Addr, Ipv6Addr},
     str::FromStr,
@@ -50,11 +50,11 @@ pub enum NetworkInterfaceKind {
 pub struct NetworkInterface {
     pub id: Uuid,
     pub kind: NetworkInterfaceKind,
-    pub name: external::Name,
+    pub name: Name,
     pub ip: IpAddr,
     pub mac: external::MacAddr,
     pub subnet: IpNet,
-    pub vni: external::Vni,
+    pub vni: Vni,
     pub primary: bool,
     pub slot: u8,
 }
@@ -624,6 +624,72 @@ impl TryFrom<&[ipnetwork::IpNetwork]> for IpAllowList {
     }
 }
 
+/// A VPC route resolved into a concrete target.
+#[derive(
+    Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct ResolvedVpcRoute {
+    pub dest: IpNet,
+    pub target: RouterTarget,
+}
+
+/// The target for a given router entry.
+#[derive(
+    Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+#[serde(tag = "type", rename_all = "snake_case", content = "value")]
+pub enum RouterTarget {
+    Drop,
+    InternetGateway,
+    Ip(IpAddr),
+    VpcSubnet(IpNet),
+}
+
+/// Information on the current parent router (and version) of a route set
+/// according to the control plane.
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct RouterVersion {
+    pub router_id: Uuid,
+    pub version: u64,
+}
+
+impl RouterVersion {
+    /// Return whether a new route set should be applied over the current
+    /// values.
+    ///
+    /// This will occur when seeing a new version and a matching parent,
+    /// or a new parent router on the control plane.
+    pub fn is_replaced_by(&self, other: &Self) -> bool {
+        (self.router_id != other.router_id) || self.version < other.version
+    }
+}
+
+/// Identifier for a VPC and/or subnet.
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct RouterId {
+    pub vni: Vni,
+    pub subnet: Option<IpNet>,
+}
+
+/// Version information for routes on a given VPC subnet.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
+pub struct ResolvedVpcRouteState {
+    pub id: RouterId,
+    pub version: Option<RouterVersion>,
+}
+
+/// An updated set of routes for a given VPC and/or subnet.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
+pub struct ResolvedVpcRouteSet {
+    pub id: RouterId,
+    pub version: Option<RouterVersion>,
+    pub routes: HashSet<ResolvedVpcRoute>,
+}
+
 #[cfg(test)]
 mod tests {
     use crate::api::internal::shared::AllowedSourceIps;

dev-tools/omdb/tests/env.out

@@ -127,6 +127,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT
@@ -251,6 +255,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.
@@ -362,6 +370,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.

dev-tools/omdb/tests/successes.out

@@ -328,6 +328,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -532,6 +536,13 @@ task: "v2p_manager"
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
 warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {})
 
+task: "vpc_route_manager"
+  configured period: every 30s
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+warning: unknown background task: "vpc_route_manager" (don't know how to interpret details: Object {})
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/

illumos-utils/src/opte/firewall_rules.rs

@@ -4,6 +4,7 @@
 
 //! Convert Omicron VPC firewall rules to OPTE firewall rules.
 
+use super::net_to_cidr;
 use crate::opte::params::VpcFirewallRule;
 use crate::opte::Vni;
 use macaddr::MacAddr6;
@@ -18,11 +19,6 @@ use oxide_vpc::api::Filters;
 use oxide_vpc::api::FirewallAction;
 use oxide_vpc::api::FirewallRule;
 use oxide_vpc::api::IpAddr;
-use oxide_vpc::api::IpCidr;
-use oxide_vpc::api::Ipv4Cidr;
-use oxide_vpc::api::Ipv4PrefixLen;
-use oxide_vpc::api::Ipv6Cidr;
-use oxide_vpc::api::Ipv6PrefixLen;
 use oxide_vpc::api::Ports;
 use oxide_vpc::api::ProtoFilter;
 use oxide_vpc::api::Protocol;
@@ -68,21 +64,10 @@ impl FromVpcFirewallRule for VpcFirewallRule {
                     HostIdentifier::Ip(IpNet::V4(net)) if net.is_host_net() => {
                         Address::Ip(IpAddr::Ip4(net.addr().into()))
                     }
-                    HostIdentifier::Ip(IpNet::V4(net)) => {
-                        Address::Subnet(IpCidr::Ip4(Ipv4Cidr::new(
-                            net.addr().into(),
-                            Ipv4PrefixLen::new(net.width()).unwrap(),
-                        )))
-                    }
                     HostIdentifier::Ip(IpNet::V6(net)) if net.is_host_net() => {
                         Address::Ip(IpAddr::Ip6(net.addr().into()))
                     }
-                    HostIdentifier::Ip(IpNet::V6(net)) => {
-                        Address::Subnet(IpCidr::Ip6(Ipv6Cidr::new(
-                            net.addr().into(),
-                            Ipv6PrefixLen::new(net.width()).unwrap(),
-                        )))
-                    }
+                    HostIdentifier::Ip(ip) => Address::Subnet(net_to_cidr(*ip)),
                     HostIdentifier::Vpc(vni) => {
                         Address::Vni(Vni::new(u32::from(*vni)).unwrap())
                     }

illumos-utils/src/opte/mod.rs

@@ -18,15 +18,23 @@ mod port;
 mod port_manager;
 
 pub use firewall_rules::opte_firewall_rules;
-pub use port::Port;
-pub use port_manager::PortManager;
-pub use port_manager::PortTicket;
-
 use ipnetwork::IpNetwork;
 use macaddr::MacAddr6;
+use omicron_common::api::internal::shared;
 pub use oxide_vpc::api::BoundaryServices;
 pub use oxide_vpc::api::DhcpCfg;
+use oxide_vpc::api::IpCidr;
+use oxide_vpc::api::Ipv4Cidr;
+use oxide_vpc::api::Ipv4PrefixLen;
+use oxide_vpc::api::Ipv6Cidr;
+use oxide_vpc::api::Ipv6PrefixLen;
+use oxide_vpc::api::RouterTarget;
 pub use oxide_vpc::api::Vni;
+use oxnet::IpNet;
+pub use port::Port;
+pub use port_manager::PortCreateParams;
+pub use port_manager::PortManager;
+pub use port_manager::PortTicket;
 use std::net::IpAddr;
 
 /// Information about the gateway for an OPTE port
@@ -63,3 +71,28 @@ impl Gateway {
         &self.ip
     }
 }
+
+/// Convert a nexus `IpNet` to an OPTE `IpCidr`.
+fn net_to_cidr(net: IpNet) -> IpCidr {
+    match net {
+        IpNet::V4(net) => IpCidr::Ip4(Ipv4Cidr::new(
+            net.addr().into(),
+            Ipv4PrefixLen::new(net.width()).unwrap(),
+        )),
+        IpNet::V6(net) => IpCidr::Ip6(Ipv6Cidr::new(
+            net.addr().into(),
+            Ipv6PrefixLen::new(net.width()).unwrap(),
+        )),
+    }
+}
+
+/// Convert a nexus `RouterTarget` to an OPTE `RouterTarget`.
+fn router_target_opte(target: &shared::RouterTarget) -> RouterTarget {
+    use shared::RouterTarget::*;
+    match target {
+        Drop => RouterTarget::Drop,
+        InternetGateway => RouterTarget::InternetGateway,
+        Ip(ip) => RouterTarget::Ip((*ip).into()),
+        VpcSubnet(net) => RouterTarget::VpcSubnet(net_to_cidr(*net)),
+    }
+}