Proxy Nexus's external API out the tech port via wicketd

common/src/address.rs

@@ -47,6 +47,16 @@ pub const CRUCIBLE_PANTRY_PORT: u16 = 17000;
 
 pub const NEXUS_INTERNAL_PORT: u16 = 12221;
 
+/// The port on which Nexus exposes its external API on the underlay network.
+///
+/// This is used by the `wicketd` Nexus proxy to allow external API access via
+/// the rack's tech port.
+pub const NEXUS_TECHPORT_EXTERNAL_PORT: u16 = 12228;
+
+/// The port on which `wicketd` runs a Nexus external API proxy on the tech port
+/// interface(s).
+pub const WICKETD_NEXUS_PROXY_PORT: u16 = 12229;
+
 pub const NTP_PORT: u16 = 123;
 
 // The number of ports available to an SNAT IP.

common/src/nexus_config.rs

@@ -5,6 +5,7 @@
 //! Configuration parameters to Nexus that are usually only known
 //! at deployment time.
 
+use crate::address::NEXUS_TECHPORT_EXTERNAL_PORT;
 use crate::api::internal::shared::SwitchLocation;
 
 use super::address::{Ipv6Subnet, RACK_PREFIX};
@@ -132,6 +133,19 @@ pub struct DeploymentConfig {
     pub id: Uuid,
     /// Uuid of the Rack where Nexus is executing.
     pub rack_id: Uuid,
+    /// Port on which the "techport external" dropshot server should listen.
+    /// This dropshot server copies _most_ of its config from
+    /// `dropshot_external` (so that it matches TLS, etc.), but builds its
+    /// listening address by combining `dropshot_internal`'s IP address with
+    /// this port.
+    ///
+    /// We use `serde(default = ...)` to ensure we don't break any serialized
+    /// configs that were created before this field was added. In production we
+    /// always expect this port to be constant, but we need to be able to
+    /// override it when running tests.
+    #[schemars(skip)]
+    #[serde(default = "default_techport_external_server_port")]
+    pub techport_external_server_port: u16,
     /// Dropshot configuration for the external API server.
     #[schemars(skip)] // TODO we're protected against dropshot changes
     pub dropshot_external: ConfigDropshotWithTls,
@@ -147,6 +161,10 @@ pub struct DeploymentConfig {
     pub external_dns_servers: Vec<IpAddr>,
 }
 
+fn default_techport_external_server_port() -> u16 {
+    NEXUS_TECHPORT_EXTERNAL_PORT
+}
+
 impl DeploymentConfig {
     /// Load a `DeploymentConfig` from the given TOML file
     ///
@@ -442,8 +460,9 @@ impl std::fmt::Display for SchemeName {
 mod test {
     use super::Tunables;
     use super::{
-        AuthnConfig, Config, ConsoleConfig, LoadError, PackageConfig,
-        SchemeName, TimeseriesDbConfig, UpdatesConfig,
+        default_techport_external_server_port, AuthnConfig, Config,
+        ConsoleConfig, LoadError, PackageConfig, SchemeName,
+        TimeseriesDbConfig, UpdatesConfig,
     };
     use crate::address::{Ipv6Subnet, RACK_PREFIX};
     use crate::api::internal::shared::SwitchLocation;
@@ -611,6 +630,8 @@ mod test {
                     rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f"
                         .parse()
                         .unwrap(),
+                    techport_external_server_port:
+                        default_techport_external_server_port(),
                     dropshot_external: ConfigDropshotWithTls {
                         tls: false,
                         dropshot: ConfigDropshot {
@@ -709,6 +730,7 @@ mod test {
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
             rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
+            techport_external_server_port = 12345
             external_dns_servers = [ "1.1.1.1", "9.9.9.9" ]
             [deployment.dropshot_external]
             bind_address = "10.1.2.3:4567"
@@ -743,6 +765,7 @@ mod test {
             config.pkg.authn.schemes_external,
             vec![SchemeName::Spoof, SchemeName::SessionCookie],
         );
+        assert_eq!(config.deployment.techport_external_server_port, 12345);
     }
 
     #[test]

gateway/Cargo.toml

@@ -9,14 +9,14 @@ anyhow.workspace = true
 async-trait.workspace = true
 ciborium.workspace = true
 clap.workspace = true
-crucible-smf.workspace = true
 dropshot.workspace = true
 futures.workspace = true
 gateway-messages.workspace = true
 gateway-sp-comms.workspace = true
 hex.workspace = true
 http.workspace = true
 hyper.workspace = true
+illumos-utils.workspace = true
 ipcc-key-value.workspace = true
 omicron-common.workspace = true
 once_cell.workspace = true

gateway/src/bin/mgs.rs

@@ -85,12 +85,11 @@ async fn do_run() -> Result<(), CmdError> {
                 ))
             })?;
 
-            let mut signals =
-                Signals::new(&[signal::SIGUSR1]).map_err(|e| {
-                    CmdError::Failure(format!(
-                        "failed to set up signal handler: {e}"
-                    ))
-                })?;
+            let mut signals = Signals::new([signal::SIGUSR1]).map_err(|e| {
+                CmdError::Failure(format!(
+                    "failed to set up signal handler: {e}"
+                ))
+            })?;
 
             let (id, addresses, rack_id) = if id_and_address_from_smf {
                 let config = read_smf_config()?;
@@ -141,7 +140,11 @@ async fn do_run() -> Result<(), CmdError> {
 
 #[cfg(target_os = "illumos")]
 fn read_smf_config() -> Result<ConfigProperties, CmdError> {
-    use crucible_smf::{Scf, ScfError};
+    fn scf_to_cmd_err(err: illumos_utils::scf::ScfError) -> CmdError {
+        CmdError::Failure(err.to_string())
+    }
+
+    use illumos_utils::scf::ScfHandle;
 
     // Name of our config property group; must match our SMF manifest.xml.
     const CONFIG_PG: &str = "config";
@@ -155,107 +158,46 @@ fn read_smf_config() -> Result<ConfigProperties, CmdError> {
     // Name of the property within CONFIG_PG for our rack ID.
     const PROP_RACK_ID: &str = "rack_id";
 
-    // This function is pretty boilerplate-y; we can reduce it by using this
-    // error type to help us construct a `CmdError::Failure(_)` string. It
-    // assumes (for the purposes of error messages) any property being fetched
-    // lives under the `CONFIG_PG` property group.
-    #[derive(Debug, thiserror::Error)]
-    enum Error {
-        #[error("failed to create scf handle: {0}")]
-        ScfHandle(ScfError),
-        #[error("failed to get self smf instance: {0}")]
-        SelfInstance(ScfError),
-        #[error("failed to get self running snapshot: {0}")]
-        RunningSnapshot(ScfError),
-        #[error("failed to get propertygroup `{CONFIG_PG}`: {0}")]
-        GetPg(ScfError),
-        #[error("missing propertygroup `{CONFIG_PG}`")]
-        MissingPg,
-        #[error("failed to get property `{CONFIG_PG}/{prop}`: {err}")]
-        GetProperty { prop: &'static str, err: ScfError },
-        #[error("missing property `{CONFIG_PG}/{prop}`")]
-        MissingProperty { prop: &'static str },
-        #[error("failed to get value for `{CONFIG_PG}/{prop}`: {err}")]
-        GetValue { prop: &'static str, err: ScfError },
-        #[error("failed to get values for `{CONFIG_PG}/{prop}`: {err}")]
-        GetValues { prop: &'static str, err: ScfError },
-        #[error("failed to get value for `{CONFIG_PG}/{prop}`")]
-        MissingValue { prop: &'static str },
-        #[error("failed to get `{CONFIG_PG}/{prop} as a string: {err}")]
-        ValueAsString { prop: &'static str, err: ScfError },
-    }
-
-    impl From<Error> for CmdError {
-        fn from(err: Error) -> Self {
-            Self::Failure(err.to_string())
-        }
-    }
-
-    let scf = Scf::new().map_err(Error::ScfHandle)?;
-    let instance = scf.get_self_instance().map_err(Error::SelfInstance)?;
-    let snapshot =
-        instance.get_running_snapshot().map_err(Error::RunningSnapshot)?;
-
-    let config = snapshot
-        .get_pg("config")
-        .map_err(Error::GetPg)?
-        .ok_or(Error::MissingPg)?;
+    let scf = ScfHandle::new().map_err(scf_to_cmd_err)?;
+    let instance = scf.self_instance().map_err(scf_to_cmd_err)?;
+    let snapshot = instance.running_snapshot().map_err(scf_to_cmd_err)?;
+    let config = snapshot.property_group(CONFIG_PG).map_err(scf_to_cmd_err)?;
 
-    let prop_id = config
-        .get_property(PROP_ID)
-        .map_err(|err| Error::GetProperty { prop: PROP_ID, err })?
-        .ok_or_else(|| Error::MissingProperty { prop: PROP_ID })?
-        .value()
-        .map_err(|err| Error::GetValue { prop: PROP_ID, err })?
-        .ok_or(Error::MissingValue { prop: PROP_ID })?
-        .as_string()
-        .map_err(|err| Error::ValueAsString { prop: PROP_ID, err })?;
+    let prop_id = config.value_as_string(PROP_ID).map_err(scf_to_cmd_err)?;
 
     let prop_id = Uuid::try_parse(&prop_id).map_err(|err| {
         CmdError::Failure(format!(
-            "failed to parse `{CONFIG_PG}/{PROP_ID}` ({prop_id:?}) as a UUID: {err}"
+            "failed to parse `{CONFIG_PG}/{PROP_ID}` \
+             ({prop_id:?}) as a UUID: {err}"
         ))
     })?;
 
-    let prop_rack_id = config
-        .get_property(PROP_RACK_ID)
-        .map_err(|err| Error::GetProperty { prop: PROP_RACK_ID, err })?
-        .ok_or_else(|| Error::MissingProperty { prop: PROP_RACK_ID })?
-        .value()
-        .map_err(|err| Error::GetValue { prop: PROP_RACK_ID, err })?
-        .ok_or(Error::MissingValue { prop: PROP_RACK_ID })?
-        .as_string()
-        .map_err(|err| Error::ValueAsString { prop: PROP_RACK_ID, err })?;
+    let prop_rack_id =
+        config.value_as_string(PROP_RACK_ID).map_err(scf_to_cmd_err)?;
 
-    let rack_id = if prop_rack_id.as_str() == "unknown" {
+    let rack_id = if prop_rack_id == "unknown" {
         None
     } else {
         Some(Uuid::try_parse(&prop_rack_id).map_err(|err| {
             CmdError::Failure(format!(
-                "failed to parse `{CONFIG_PG}/{PROP_RACK_ID}` ({prop_rack_id:?}) as a UUID: {err}"
+                "failed to parse `{CONFIG_PG}/{PROP_RACK_ID}` \
+                 ({prop_rack_id:?}) as a UUID: {err}"
             ))
         })?)
     };
 
-    let prop_addr = config
-        .get_property(PROP_ADDR)
-        .map_err(|err| Error::GetProperty { prop: PROP_ADDR, err })?
-        .ok_or_else(|| Error::MissingProperty { prop: PROP_ADDR })?;
+    let prop_addr =
+        config.values_as_strings(PROP_ADDR).map_err(scf_to_cmd_err)?;
 
-    let mut addresses = Vec::new();
+    let mut addresses = Vec::with_capacity(prop_addr.len());
 
-    for value in prop_addr
-        .values()
-        .map_err(|err| Error::GetValues { prop: PROP_ADDR, err })?
-    {
-        let addr = value
-            .map_err(|err| Error::GetValue { prop: PROP_ADDR, err })?
-            .as_string()
-            .map_err(|err| Error::ValueAsString { prop: PROP_ADDR, err })?;
-
-        addresses.push(addr.parse().map_err(|err| CmdError::Failure(format!(
-            "failed to parse `{CONFIG_PG}/{PROP_ADDR}` ({addr:?}) as a socket address: {err}"
-        )))?);
+    for addr in prop_addr {
+        addresses.push(addr.parse().map_err(|err| {
+            CmdError::Failure(format!(
+                "failed to parse `{CONFIG_PG}/{PROP_ADDR}` \
+             ({addr:?}) as a socket address: {err}"
+            ))
+        })?);
     }
 
     if addresses.is_empty() {

illumos-utils/Cargo.toml

@@ -12,6 +12,7 @@ bhyve_api.workspace = true
 byteorder.workspace = true
 camino.workspace = true
 cfg-if.workspace = true
+crucible-smf.workspace = true
 futures.workspace = true
 ipnetwork.workspace = true
 libc.workspace = true

illumos-utils/src/lib.rs

@@ -17,6 +17,7 @@ pub mod libc;
 pub mod link;
 pub mod opte;
 pub mod running_zone;
+pub mod scf;
 pub mod svc;
 pub mod vmm_reservoir;
 pub mod zfs;