diff --git a/Cargo.lock b/Cargo.lock index 923693276a9..0359ec2d044 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3887,7 +3887,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=4830268f642f766180dee8cbafdca790916bfa09#4830268f642f766180dee8cbafdca790916bfa09" [[package]] name = "illumos-utils" @@ -4323,7 +4323,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=4830268f642f766180dee8cbafdca790916bfa09#4830268f642f766180dee8cbafdca790916bfa09" dependencies = [ "quote", "syn 2.0.74", @@ -6863,7 +6863,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=4830268f642f766180dee8cbafdca790916bfa09#4830268f642f766180dee8cbafdca790916bfa09" dependencies = [ "cfg-if", "dyn-clone", @@ -6880,7 +6880,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=4830268f642f766180dee8cbafdca790916bfa09#4830268f642f766180dee8cbafdca790916bfa09" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -6892,7 +6892,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=4830268f642f766180dee8cbafdca790916bfa09#4830268f642f766180dee8cbafdca790916bfa09" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -6966,7 +6966,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=4830268f642f766180dee8cbafdca790916bfa09#4830268f642f766180dee8cbafdca790916bfa09" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 3d1d19fa65b..8cc38a7c34f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -458,7 +458,7 @@ omicron-test-utils = { path = "test-utils" } omicron-workspace-hack = "0.1.0" omicron-zone-package = "0.11.0" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "76878de67229ea113d70503c441eab47ac5dc653", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "4830268f642f766180dee8cbafdca790916bfa09", features = [ "api", "std" ] } oxlog = { path = "dev-tools/oxlog" } oxnet = { git = "https://github.com/oxidecomputer/oxnet" } once_cell = "1.19.0" @@ -468,7 +468,7 @@ openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "76878de67229ea113d70503c441eab47ac5dc653" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "4830268f642f766180dee8cbafdca790916bfa09" } oso = "0.27" owo-colors = "4.0.0" oximeter = { path = "oximeter/oximeter" } diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 4826292863a..b3da4dae504 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -777,7 +777,7 @@ pub struct DhcpConfig { #[serde(tag = "type", rename_all = "snake_case", content = "value")] pub enum RouterTarget { Drop, - InternetGateway, + InternetGateway(IpAddr), Ip(IpAddr), VpcSubnet(IpNet), } diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 9a86711ae62..d141ab6cec9 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -90,7 +90,7 @@ fn router_target_opte(target: &shared::RouterTarget) -> RouterTarget { use shared::RouterTarget::*; match target { Drop => RouterTarget::Drop, - InternetGateway => RouterTarget::InternetGateway, + InternetGateway(ip) => RouterTarget::InternetGateway((*ip).into()), Ip(ip) => RouterTarget::Ip((*ip).into()), VpcSubnet(net) => RouterTarget::VpcSubnet(net_to_cidr(*net)), } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 735428907e4..b473b00896b 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -46,6 +46,7 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; use std::net::IpAddr; +use std::net::Ipv4Addr; use std::net::Ipv6Addr; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; @@ -249,7 +250,7 @@ impl PortManager { }; let vpc_cfg = VpcCfg { - ip_cfg, + ip_cfg: ip_cfg.clone(), guest_mac: MacAddr::from(nic.mac.into_array()), gateway_mac: MacAddr::from(gateway.mac.into_array()), vni, @@ -329,6 +330,19 @@ impl PortManager { // create a record to show that we're interested in receiving // those routes. let mut routes = self.inner.routes.lock().unwrap(); + let system_routes = match &ip_cfg { + IpCfg::Ipv4(cfg) => { + system_routes_v4(cfg, is_service, &mut routes, &port) + } + IpCfg::Ipv6(cfg) => { + system_routes_v6(cfg, is_service, &mut routes, &port) + } + IpCfg::DualStack { ipv4, ipv6 } => { + system_routes_v4(ipv4, is_service, &mut routes, &port); + system_routes_v6(ipv6, is_service, &mut routes, &port) + } + }; + /* let system_routes = routes.entry(port.system_router_key()).or_insert_with(|| { let mut routes = HashSet::new(); @@ -349,6 +363,7 @@ impl PortManager { RouteSet { version: None, routes, active_ports: 0 } }); + */ system_routes.active_ports += 1; // Clone is needed to get borrowck on our side, sadly. let system_routes = system_routes.clone(); @@ -939,3 +954,87 @@ impl Drop for PortTicket { let _ = self.release_inner(); } } + +fn system_routes_v4<'a>( + cfg: &Ipv4Cfg, + is_service: bool, + routes: &'a mut HashMap, + port: &Port, +) -> &'a mut RouteSet { + routes.entry(port.system_router_key()).or_insert_with(|| { + let mut routes = HashSet::new(); + if let Some(ref snat) = cfg.external_ips.snat { + if is_service { + routes.insert(ResolvedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway(IpAddr::V4( + Ipv4Addr::from(snat.external_ip), + )), + }); + } + } + if let Some(ref ephemeral) = cfg.external_ips.ephemeral_ip { + if is_service { + routes.insert(ResolvedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway(IpAddr::V4( + Ipv4Addr::from(*ephemeral), + )), + }); + } + } + if is_service { + for fip in &cfg.external_ips.floating_ips { + routes.insert(ResolvedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway(IpAddr::V4( + Ipv4Addr::from(*fip), + )), + }); + } + } + RouteSet { version: None, routes, active_ports: 0 } + }) +} + +fn system_routes_v6<'a>( + cfg: &Ipv6Cfg, + is_service: bool, + routes: &'a mut HashMap, + port: &Port, +) -> &'a mut RouteSet { + routes.entry(port.system_router_key()).or_insert_with(|| { + let mut routes = HashSet::new(); + if let Some(ref snat) = cfg.external_ips.snat { + if is_service { + routes.insert(ResolvedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway(IpAddr::V6( + Ipv6Addr::from(snat.external_ip), + )), + }); + } + } + if let Some(ref ephemeral) = cfg.external_ips.ephemeral_ip { + if is_service { + routes.insert(ResolvedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway(IpAddr::V6( + Ipv6Addr::from(*ephemeral), + )), + }); + } + } + if is_service { + for fip in &cfg.external_ips.floating_ips { + routes.insert(ResolvedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway(IpAddr::V6( + Ipv6Addr::from(*fip), + )), + }); + } + } + RouteSet { version: None, routes, active_ports: 0 } + }) +} diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index e8123e09a08..6395abeaddc 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -2413,12 +2413,12 @@ impl DataStore { RouteDestination::Vpc(_) => (None, None), }; - let (v4_target, v6_target) = match rule.target.0 { + let (v4_target, v6_target) = match &rule.target.0 { RouteTarget::Ip(ip @ IpAddr::V4(_)) => { - (Some(RouterTarget::Ip(ip)), None) + (Some(RouterTarget::Ip(*ip)), None) } RouteTarget::Ip(ip @ IpAddr::V6(_)) => { - (None, Some(RouterTarget::Ip(ip))) + (None, Some(RouterTarget::Ip(*ip))) } RouteTarget::Subnet(n) => subnets .get(&n) @@ -2449,15 +2449,9 @@ impl DataStore { (Some(RouterTarget::Drop), Some(RouterTarget::Drop)) } - // TODO: Internet Gateways. - // The semantic here is 'name match => allow', - // as the other aspect they will control is SNAT - // IP allocation. Today, presence of this rule - // allows upstream regardless of name. - RouteTarget::InternetGateway(_n) => ( - Some(RouterTarget::InternetGateway), - Some(RouterTarget::InternetGateway), - ), + // There can be multiple targets per internet gateway, so these + // are handled below. + RouteTarget::InternetGateway(_) => (None, None), // TODO: VPC Peering. RouteTarget::Vpc(_) => (None, None), @@ -2476,6 +2470,126 @@ impl DataStore { if let (Some(dest), Some(target)) = (v6_dest, v6_target) { out.insert(dest, target); } + + // The OPTE model for internet gateway routes is that a VPC + // route points at an internet gateway object. That internet + // gateway object contains the source address that is to be + // used for the route. + // + // Therefore, here what we are doing is ... + // 1. Look up the intergnet gateway (igw). + // 2. Fetch the IP pools associated with the igw. + // 3. Look up the external IPs in the vpc. + // 4. For each external IP, see if it belongs to an IP pool + // associated with an internet gateway (yeah, this is + // quadratic, but the number of ip pool associations is + // unlikely to be more than a couple?) + // 5. If the external IP does belong to an IP pool that is + // associated with the gateway target, install the + // destination -> target rule where the target is the + // internet gateway parameterized by the external ip. + if let RouteTarget::InternetGateway(name) = &rule.target.0 { + let conn = self.pool_connection_authorized(opctx).await?; + let igw = db::lookup::LookupPath::new(opctx, self) + .vpc_id(authz_vpc.id()) + .internet_gateway_name(&(name.clone().into())) + .fetch() + .await + .ok(); + + let (.., authz_igw, _db_igw) = match igw { + Some(value) => value, + None => return Ok(out), + }; + + use db::schema::internet_gateway_ip_pool::dsl as igwp; + let igw_pools = igwp::internet_gateway_ip_pool + .filter(igwp::time_deleted.is_null()) + .filter(igwp::internet_gateway_id.eq(authz_igw.id())) + .select(InternetGatewayIpPool::as_select()) + .load_async::(&*conn) + .await + .ok(); + + let igw_pools = match igw_pools { + Some(value) => value, + None => return Ok(out), + }; + + for igw_pool in &igw_pools { + use db::schema::ip_pool_range::dsl as ipr; + let prs = match ipr::ip_pool_range + .filter(ipr::time_deleted.is_null()) + .filter(ipr::ip_pool_id.eq(igw_pool.ip_pool_id)) + .select(IpPoolRange::as_select()) + .load_async::(&*conn) + .await + { + Ok(value) => value, + Err(_) => continue, + }; + + for x in instances.values() { + let ifx = &x.1; + use db::schema::external_ip::dsl as xip; + let ext_ips = xip::external_ip + .filter(xip::time_deleted.is_null()) + .filter(xip::parent_id.eq(ifx.instance_id)) + .select(ExternalIp::as_select()) + .load_async::(&*conn) + .await + .ok(); + + let ext_ips = match ext_ips { + Some(value) => value, + None => continue, + }; + + for ext_ip in &ext_ips { + match ext_ip.ip.ip() { + IpAddr::V4(v4) => { + if let Some(dest) = v4_dest { + for pr in &prs { + if ext_ip.ip.ip() + >= pr.first_address.ip() + && ext_ip.ip.ip() + <= pr.last_address.ip() + { + out.insert( + dest, + RouterTarget::InternetGateway( + v4.into(), + ), + ); + break; + } + } + } + } + IpAddr::V6(v6) => { + if let Some(dest) = v6_dest { + for pr in &prs { + if ext_ip.ip.ip() + >= pr.first_address.ip() + && ext_ip.ip.ip() + <= pr.last_address.ip() + { + out.insert( + dest, + RouterTarget::InternetGateway( + v6.into(), + ), + ); + break; + } + } + } + } + }; + } + } + } + } } Ok(out) diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index bb8e4e0b87e..8253fba5bdb 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -4710,10 +4710,15 @@ "enum": [ "internet_gateway" ] + }, + "value": { + "type": "string", + "format": "ip" } }, "required": [ - "type" + "type", + "value" ] }, {