Compat with newer OPTE, more believable router rules

common/src/api/external/mod.rs

@@ -1675,6 +1675,9 @@ pub enum RouteTarget {
     #[display("inetgw:{0}")]
     /// Forward traffic to an internet gateway
     InternetGateway(Name),
+    #[display("drop")]
+    /// Drop matching traffic
+    Drop,
 }
 
 /// A `RouteDestination` is used to match traffic with a routing rule, on the

illumos-utils/src/opte/port_manager.rs

@@ -25,6 +25,7 @@ use oxide_vpc::api::IpCidr;
 use oxide_vpc::api::Ipv4Cfg;
 use oxide_vpc::api::Ipv6Cfg;
 use oxide_vpc::api::MacAddr;
+use oxide_vpc::api::RouterClass;
 use oxide_vpc::api::RouterTarget;
 use oxide_vpc::api::SNat4Cfg;
 use oxide_vpc::api::SNat6Cfg;
@@ -339,9 +340,16 @@ impl PortManager {
             (port, ticket)
         };
 
+        // TODO: These should not be filled in like this, and should be informed
+        //       by either our existing knowledge of current knowledge of system + custom
+        //       routers OR we just await the router RPW filling this in for us.
+        //       In future, ∃ VPCs *without* an Internet Gateway so we can't just
+        //       plumb that in as well...
+
         // Add a router entry for this interface's subnet, directing traffic to the
         // VPC subnet.
         let route = AddRouterEntryReq {
+            class: RouterClass::System,
             port_name: port_name.clone(),
             dest: vpc_subnet,
             target: RouterTarget::VpcSubnet(vpc_subnet),
@@ -378,6 +386,7 @@ impl PortManager {
         .parse()
         .unwrap();
         let route = AddRouterEntryReq {
+            class: RouterClass::System,
             port_name: port_name.clone(),
             dest,
             target: RouterTarget::InternetGateway,

nexus/db-model/src/schema.rs

@@ -1100,6 +1100,7 @@ table! {
         rcgen -> Int8,
         ipv4_block -> Inet,
         ipv6_block -> Inet,
+        custom_router_id -> Nullable<Uuid>,
     }
 }
 

nexus/db-model/src/schema_versions.rs

@@ -17,7 +17,7 @@ use std::collections::BTreeMap;
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(61, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(62, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy<Vec<KnownVersion>> = Lazy::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(62, "vpc-subnet-routing"),
         KnownVersion::new(61, "blueprint-add-sled-state"),
         KnownVersion::new(60, "add-lookup-vmm-by-sled-id-index"),
         KnownVersion::new(59, "enforce-first-as-default"),

nexus/db-model/src/vpc_subnet.rs

@@ -39,6 +39,7 @@ pub struct VpcSubnet {
     pub rcgen: Generation,
     pub ipv4_block: Ipv4Net,
     pub ipv6_block: Ipv6Net,
+    pub custom_router_id: Option<Uuid>,
 }
 
 impl VpcSubnet {
@@ -60,6 +61,7 @@ impl VpcSubnet {
             rcgen: Generation::new(),
             ipv4_block: Ipv4Net(ipv4_block),
             ipv6_block: Ipv6Net(ipv6_block),
+            custom_router_id: None,
         }
     }
 
@@ -102,6 +104,7 @@ impl From<VpcSubnet> for views::VpcSubnet {
             vpc_id: subnet.vpc_id,
             ipv4_block: subnet.ipv4_block.0,
             ipv6_block: subnet.ipv6_block.0,
+            custom_router_id: subnet.custom_router_id,
         }
     }
 }

nexus/db-queries/src/db/datastore/vpc.rs

@@ -74,7 +74,8 @@ impl DataStore {
     ) -> Result<(), Error> {
         use crate::db::fixed_data::project::SERVICES_PROJECT_ID;
         use crate::db::fixed_data::vpc::SERVICES_VPC;
-        use crate::db::fixed_data::vpc::SERVICES_VPC_DEFAULT_ROUTE_ID;
+        use crate::db::fixed_data::vpc::SERVICES_VPC_DEFAULT_V4_ROUTE_ID;
+        use crate::db::fixed_data::vpc::SERVICES_VPC_DEFAULT_V6_ROUTE_ID;
 
         opctx.authorize(authz::Action::Modify, &authz::DATABASE).await?;
 
@@ -135,35 +136,41 @@ impl DataStore {
                 .map(|(authz_router, _)| authz_router)?
         };
 
-        let route = RouterRoute::new(
-            *SERVICES_VPC_DEFAULT_ROUTE_ID,
-            SERVICES_VPC.system_router_id,
-            RouterRouteKind::Default,
-            nexus_types::external_api::params::RouterRouteCreate {
-                identity: IdentityMetadataCreateParams {
-                    name: "default".parse().unwrap(),
-                    description:
-                        "Default internet gateway route for Oxide Services"
-                            .to_string(),
+        // Unwrap safety: these are known valid CIDR blocks.
+        let default_ips = [
+            ("0.0.0.0/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V4_ROUTE_ID),
+            ("::/0".parse().unwrap(), *SERVICES_VPC_DEFAULT_V6_ROUTE_ID),
+        ];
+
+        for (default, uuid) in default_ips {
+            let route = RouterRoute::new(
+                uuid,
+                SERVICES_VPC.system_router_id,
+                RouterRouteKind::Default,
+                nexus_types::external_api::params::RouterRouteCreate {
+                    identity: IdentityMetadataCreateParams {
+                        name: "default".parse().unwrap(),
+                        description:
+                            "Default internet gateway route for Oxide Services"
+                                .to_string(),
+                    },
+                    target: RouteTarget::InternetGateway(
+                        "outbound".parse().unwrap(),
+                    ),
+                    destination: RouteDestination::IpNet(default),
                 },
-                target: RouteTarget::InternetGateway(
-                    "outbound".parse().unwrap(),
-                ),
-                destination: RouteDestination::Vpc(
-                    SERVICES_VPC.identity.name.clone().into(),
-                ),
-            },
-        );
-        self.router_create_route(opctx, &authz_router, route)
-            .await
-            .map(|_| ())
-            .or_else(|e| match e {
-                Error::ObjectAlreadyExists { .. } => Ok(()),
-                _ => Err(e),
-            })?;
+            );
+            self.router_create_route(opctx, &authz_router, route)
+                .await
+                .map(|_| ())
+                .or_else(|e| match e {
+                    Error::ObjectAlreadyExists { .. } => Ok(()),
+                    _ => Err(e),
+                })?;
+        }
 
         self.load_builtin_vpc_fw_rules(opctx).await?;
-        self.load_builtin_vpc_subnets(opctx).await?;
+        self.load_builtin_vpc_subnets(opctx, &authz_router).await?;
 
         info!(opctx.log, "created built-in services vpc");
 
@@ -228,10 +235,14 @@ impl DataStore {
     async fn load_builtin_vpc_subnets(
         &self,
         opctx: &OpContext,
+        authz_router: &authz::VpcRouter,
     ) -> Result<(), Error> {
         use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET;
+        use crate::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET_ROUTE_ID;
         use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET;
+        use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET_ROUTE_ID;
         use crate::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET;
+        use crate::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET_ROUTE_ID;
 
         debug!(opctx.log, "attempting to create built-in VPC Subnets");
 
@@ -242,9 +253,11 @@ impl DataStore {
             .lookup_for(authz::Action::CreateChild)
             .await
             .internal_context("lookup built-in services vpc")?;
-        for vpc_subnet in
-            [&*DNS_VPC_SUBNET, &*NEXUS_VPC_SUBNET, &*NTP_VPC_SUBNET]
-        {
+        for (vpc_subnet, route_id) in [
+            (&*DNS_VPC_SUBNET, *DNS_VPC_SUBNET_ROUTE_ID),
+            (&*NEXUS_VPC_SUBNET, *NEXUS_VPC_SUBNET_ROUTE_ID),
+            (&*NTP_VPC_SUBNET, *NTP_VPC_SUBNET_ROUTE_ID),
+        ] {
             if let Ok(_) = db::lookup::LookupPath::new(opctx, self)
                 .vpc_subnet_id(vpc_subnet.id())
                 .fetch()
@@ -260,6 +273,31 @@ impl DataStore {
                     Error::ObjectAlreadyExists { .. } => Ok(()),
                     _ => Err(e),
                 })?;
+
+            let route = RouterRoute::new(
+                route_id,
+                *SERVICES_VPC_ID,
+                RouterRouteKind::Default,
+                nexus_types::external_api::params::RouterRouteCreate {
+                    identity: IdentityMetadataCreateParams {
+                        name: "default".parse().unwrap(),
+                        description:
+                            "Default internet gateway route for Oxide Services"
+                                .to_string(),
+                    },
+                    target: RouteTarget::Subnet(vpc_subnet.name().clone()),
+                    destination: RouteDestination::Subnet(
+                        vpc_subnet.name().clone(),
+                    ),
+                },
+            );
+            self.router_create_route(opctx, &authz_router, route)
+                .await
+                .map(|_| ())
+                .or_else(|e| match e {
+                    Error::ObjectAlreadyExists { .. } => Ok(()),
+                    _ => Err(e),
+                })?;
         }
 
         info!(opctx.log, "created built-in services vpc subnets");

nexus/db-queries/src/db/fixed_data/vpc.rs

@@ -24,11 +24,19 @@ pub static SERVICES_VPC_ROUTER_ID: Lazy<uuid::Uuid> = Lazy::new(|| {
 });
 
 /// UUID of default route for built-in Services VPC.
-pub static SERVICES_VPC_DEFAULT_ROUTE_ID: Lazy<uuid::Uuid> = Lazy::new(|| {
-    "001de000-074c-4000-8000-000000000002"
-        .parse()
-        .expect("invalid uuid for builtin services vpc default route id")
-});
+pub static SERVICES_VPC_DEFAULT_V4_ROUTE_ID: Lazy<uuid::Uuid> =
+    Lazy::new(|| {
+        "001de000-074c-4000-8000-000000000002"
+            .parse()
+            .expect("invalid uuid for builtin services vpc default route id")
+    });
+
+pub static SERVICES_VPC_DEFAULT_V6_ROUTE_ID: Lazy<uuid::Uuid> =
+    Lazy::new(|| {
+        "001de000-074c-4000-8000-000000000003"
+            .parse()
+            .expect("invalid uuid for builtin services vpc default route id")
+    });
 
 /// Built-in VPC for internal services on the rack.
 pub static SERVICES_VPC: Lazy<db::model::IncompleteVpc> = Lazy::new(|| {

nexus/db-queries/src/db/fixed_data/vpc_subnet.rs

@@ -31,6 +31,24 @@ pub static NTP_VPC_SUBNET_ID: Lazy<uuid::Uuid> = Lazy::new(|| {
         .expect("invalid uuid for builtin boundary ntp vpc subnet id")
 });
 
+pub static DNS_VPC_SUBNET_ROUTE_ID: Lazy<uuid::Uuid> = Lazy::new(|| {
+    "001de000-c470-4000-8000-000000000004"
+        .parse()
+        .expect("invalid uuid for builtin services vpc default route id")
+});
+
+pub static NEXUS_VPC_SUBNET_ROUTE_ID: Lazy<uuid::Uuid> = Lazy::new(|| {
+    "001de000-c470-4000-8000-000000000005"
+        .parse()
+        .expect("invalid uuid for builtin services vpc default route id")
+});
+
+pub static NTP_VPC_SUBNET_ROUTE_ID: Lazy<uuid::Uuid> = Lazy::new(|| {
+    "001de000-c470-4000-8000-000000000006"
+        .parse()
+        .expect("invalid uuid for builtin services vpc default route id")
+});
+
 /// Built-in VPC Subnet for External DNS.
 pub static DNS_VPC_SUBNET: Lazy<VpcSubnet> = Lazy::new(|| {
     VpcSubnet::new(