plumb customer-configured DNS into instances

illumos-utils/src/opte/mod.rs

@@ -25,6 +25,7 @@ pub use port_manager::PortTicket;
 use ipnetwork::IpNetwork;
 use macaddr::MacAddr6;
 pub use oxide_vpc::api::BoundaryServices;
+pub use oxide_vpc::api::DhcpCfg;
 pub use oxide_vpc::api::Vni;
 use std::net::IpAddr;
 

illumos-utils/src/opte/params.rs

@@ -50,3 +50,26 @@ pub struct DeleteVirtualNetworkInterfaceHost {
     /// be deleted.
     pub vni: external::Vni,
 }
+
+/// DHCP configuration for a port
+///
+/// Not present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we
+/// use `InstanceRuntimeState::hostname` for this value.
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+pub struct DhcpConfig {
+    /// DNS servers to send to the instance
+    ///
+    /// (DHCPv4 option 6; DHCPv6 option 23)
+    pub dns_servers: Vec<IpAddr>,
+
+    /// DNS zone this instance's hostname belongs to (e.g. the `project.example`
+    /// part of `instance1.project.example`)
+    ///
+    /// (DHCPv4 option 15; used in DHCPv6 option 39)
+    pub host_domain: Option<String>,
+
+    /// DNS search domains
+    ///
+    /// (DHCPv4 option 119; DHCPv6 option 24)
+    pub search_domains: Vec<String>,
+}

illumos-utils/src/opte/port_manager.rs

@@ -19,6 +19,7 @@ use omicron_common::api::internal::shared::NetworkInterface;
 use omicron_common::api::internal::shared::NetworkInterfaceKind;
 use omicron_common::api::internal::shared::SourceNatConfig;
 use oxide_vpc::api::AddRouterEntryReq;
+use oxide_vpc::api::DhcpCfg;
 use oxide_vpc::api::IpCfg;
 use oxide_vpc::api::IpCidr;
 use oxide_vpc::api::Ipv4Cfg;
@@ -100,6 +101,7 @@ impl PortManager {
         source_nat: Option<SourceNatConfig>,
         external_ips: &[IpAddr],
         firewall_rules: &[VpcFirewallRule],
+        dhcp_config: DhcpCfg,
     ) -> Result<(Port, PortTicket), Error> {
         let mac = *nic.mac;
         let vni = Vni::new(nic.vni).unwrap();
@@ -205,8 +207,6 @@ impl PortManager {
             vni,
             phys_ip: self.inner.underlay_ip.into(),
             boundary_services,
-            // TODO-completeness (#2153): Plumb domain search list
-            domain_list: vec![],
         };
 
         // Create the xde device.
@@ -235,6 +235,19 @@ impl PortManager {
             hdl
         };
 
+        // Configure the DHCP parameters for this interface.
+        debug!(
+            self.inner.log,
+            "Set DHCP parameters";
+            "port_name" => &port_name,
+            "dhcp_config" => ?&dhcp_config,
+        );
+        #[cfg(target_os = "illumos")]
+        hdl.set_dhcp_params(&oxide_vpc::api::SetDhcpParamsReq {
+            port_name: port_name.clone(),
+            data: dhcp_config,
+        })?;
+
         // Initialize firewall rules for the new port.
         let rules = opte_firewall_rules(firewall_rules, &vni, &mac);
         debug!(

nexus/src/app/instance.rs

@@ -812,6 +812,12 @@ impl super::Nexus {
             source_nat,
             external_ips,
             firewall_rules,
+            dhcp_config: sled_agent_client::types::DhcpConfig {
+                dns_servers: self.external_dns_servers.clone(),
+                // TODO: finish designing instance DNS
+                host_domain: None,
+                search_domains: Vec::new(),
+            },
             disks: disk_reqs,
             cloud_init_bytes: Some(base64::Engine::encode(
                 &base64::engine::general_purpose::STANDARD,

nexus/src/app/mod.rs

@@ -26,7 +26,7 @@ use omicron_common::api::internal::shared::SwitchLocation;
 use omicron_common::nexus_config::RegionAllocationStrategy;
 use slog::Logger;
 use std::collections::HashMap;
-use std::net::Ipv6Addr;
+use std::net::{IpAddr, Ipv6Addr};
 use std::sync::Arc;
 use uuid::Uuid;
 
@@ -149,6 +149,12 @@ pub struct Nexus {
     /// DNS resolver Nexus uses to resolve an external host
     external_resolver: Arc<external_dns::Resolver>,
 
+    /// DNS servers used in `external_resolver`, used to provide DNS servers to
+    /// instances via DHCP
+    // TODO: This needs to be moved to the database.
+    // https://github.com/oxidecomputer/omicron/issues/3732
+    external_dns_servers: Vec<IpAddr>,
+
     /// Mapping of SwitchLocations to their respective Dendrite Clients
     dpd_clients: HashMap<SwitchLocation, Arc<dpd_client::Client>>,
 
@@ -327,6 +333,10 @@ impl Nexus {
             samael_max_issue_delay: std::sync::Mutex::new(None),
             internal_resolver: resolver,
             external_resolver,
+            external_dns_servers: config
+                .deployment
+                .external_dns_servers
+                .clone(),
             dpd_clients,
             background_tasks,
             default_region_allocation_strategy: config

openapi/sled-agent.json

@@ -1194,6 +1194,36 @@
           "vni"
         ]
       },
+      "DhcpConfig": {
+        "description": "DHCP configuration for a port\n\nNot present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we use `InstanceRuntimeState::hostname` for this value.",
+        "type": "object",
+        "properties": {
+          "dns_servers": {
+            "description": "DNS servers to send to the instance\n\n(DHCPv4 option 6; DHCPv6 option 23)",
+            "type": "array",
+            "items": {
+              "type": "string",
+              "format": "ip"
+            }
+          },
+          "host_domain": {
+            "nullable": true,
+            "description": "DNS zone this instance's hostname belongs to (e.g. the `project.example` part of `instance1.project.example`)\n\n(DHCPv4 option 15; used in DHCPv6 option 39)",
+            "type": "string"
+          },
+          "search_domains": {
+            "description": "DNS search domains\n\n(DHCPv4 option 119; DHCPv6 option 24)",
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          }
+        },
+        "required": [
+          "dns_servers",
+          "search_domains"
+        ]
+      },
       "DiskEnsureBody": {
         "description": "Sent from to a sled agent to establish the runtime state of a Disk",
         "type": "object",
@@ -1668,6 +1698,9 @@
             "nullable": true,
             "type": "string"
           },
+          "dhcp_config": {
+            "$ref": "#/components/schemas/DhcpConfig"
+          },
           "disks": {
             "type": "array",
             "items": {
@@ -1702,6 +1735,7 @@
           }
         },
         "required": [
+          "dhcp_config",
           "disks",
           "external_ips",
           "firewall_rules",

sled-agent/Cargo.toml

@@ -44,7 +44,6 @@ macaddr.workspace = true
 nexus-client.workspace = true
 omicron-common.workspace = true
 once_cell.workspace = true
-oxide-vpc.workspace = true
 oximeter.workspace = true
 oximeter-producer.workspace = true
 percent-encoding.workspace = true

sled-agent/src/instance.rs

@@ -25,7 +25,7 @@ use backoff::BackoffError;
 use futures::lock::{Mutex, MutexGuard};
 use illumos_utils::dladm::Etherstub;
 use illumos_utils::link::VnicAllocator;
-use illumos_utils::opte::PortManager;
+use illumos_utils::opte::{DhcpCfg, PortManager};
 use illumos_utils::running_zone::{InstalledZone, RunningZone};
 use illumos_utils::svc::wait_for_service;
 use illumos_utils::zone::Zones;
@@ -89,6 +89,10 @@ pub enum Error {
     #[error(transparent)]
     Opte(#[from] illumos_utils::opte::Error),
 
+    /// Issued by `impl TryFrom<&[u8]> for oxide_vpc::api::DomainName`
+    #[error("Invalid hostname: {0}")]
+    InvalidHostname(&'static str),
+
     #[error("Error resolving DNS name: {0}")]
     ResolveError(#[from] internal_dns::resolver::ResolveError),
 
@@ -229,6 +233,7 @@ struct InstanceInner {
     source_nat: SourceNatConfig,
     external_ips: Vec<IpAddr>,
     firewall_rules: Vec<VpcFirewallRule>,
+    dhcp_config: DhcpCfg,
 
     // Disk related properties
     // TODO: replace `propolis_client::handmade::*` with properly-modeled local types
@@ -626,6 +631,38 @@ impl Instance {
         zone_bundler: ZoneBundler,
     ) -> Result<Self, Error> {
         info!(log, "Instance::new w/initial HW: {:?}", initial);
+
+        let mut dhcp_config = DhcpCfg {
+            hostname: Some(
+                initial
+                    .runtime
+                    .hostname
+                    .parse()
+                    .map_err(Error::InvalidHostname)?,
+            ),
+            host_domain: initial
+                .dhcp_config
+                .host_domain
+                .map(|domain| domain.parse())
+                .transpose()
+                .map_err(Error::InvalidHostname)?,
+            domain_search_list: initial
+                .dhcp_config
+                .search_domains
+                .into_iter()
+                .map(|domain| domain.parse())
+                .collect::<Result<_, _>>()
+                .map_err(Error::InvalidHostname)?,
+            dns4_servers: Vec::new(),
+            dns6_servers: Vec::new(),
+        };
+        for ip in initial.dhcp_config.dns_servers {
+            match ip {
+                IpAddr::V4(ip) => dhcp_config.dns4_servers.push(ip.into()),
+                IpAddr::V6(ip) => dhcp_config.dns6_servers.push(ip.into()),
+            }
+        }
+
         let instance = InstanceInner {
             log: log.new(o!("instance_id" => id.to_string())),
             // NOTE: Mostly lies.
@@ -649,6 +686,7 @@ impl Instance {
             source_nat: initial.source_nat,
             external_ips: initial.external_ips,
             firewall_rules: initial.firewall_rules,
+            dhcp_config,
             requested_disks: initial.disks,
             cloud_init_bytes: initial.cloud_init_bytes,
             state: InstanceStates::new(initial.runtime),
@@ -879,6 +917,7 @@ impl Instance {
                 snat,
                 external_ips,
                 &inner.firewall_rules,
+                inner.dhcp_config.clone(),
             )?;
             opte_ports.push(port);
         }

sled-agent/src/params.rs

@@ -6,6 +6,7 @@ use crate::zone_bundle::PriorityOrder;
 pub use crate::zone_bundle::ZoneBundleCause;
 pub use crate::zone_bundle::ZoneBundleId;
 pub use crate::zone_bundle::ZoneBundleMetadata;
+pub use illumos_utils::opte::params::DhcpConfig;
 pub use illumos_utils::opte::params::VpcFirewallRule;
 pub use illumos_utils::opte::params::VpcFirewallRulesEnsureBody;
 use omicron_common::api::internal::nexus::{
@@ -67,6 +68,7 @@ pub struct InstanceHardware {
     /// provided to an instance to allow inbound connectivity.
     pub external_ips: Vec<IpAddr>,
     pub firewall_rules: Vec<VpcFirewallRule>,
+    pub dhcp_config: DhcpConfig,
     // TODO: replace `propolis_client::handmade::*` with locally-modeled request type
     pub disks: Vec<propolis_client::handmade::api::DiskRequest>,
     pub cloud_init_bytes: Option<String>,

sled-agent/src/services.rs

@@ -52,7 +52,7 @@ use illumos_utils::dladm::{
     Dladm, Etherstub, EtherstubVnic, GetSimnetError, PhysicalLink,
 };
 use illumos_utils::link::{Link, VnicAllocator};
-use illumos_utils::opte::{Port, PortManager, PortTicket};
+use illumos_utils::opte::{DhcpCfg, Port, PortManager, PortTicket};
 use illumos_utils::running_zone::{
     InstalledZone, RunCommandError, RunningZone,
 };
@@ -862,11 +862,11 @@ impl ServiceManager {
             // config allows outbound access which is enough for
             // Boundary NTP which needs to come up before Nexus.
             let port = port_manager
-                .create_port(nic, snat, external_ips, &[])
+                .create_port(nic, snat, external_ips, &[], DhcpCfg::default())
                 .map_err(|err| Error::ServicePortCreation {
-                service: svc.details.to_string(),
-                err: Box::new(err),
-            })?;
+                    service: svc.details.to_string(),
+                    err: Box::new(err),
+                })?;
 
             // We also need to update the switch with the NAT mappings
             let (target_ip, first_port, last_port) = match snat {