Merge branch 'main' into dendrite-integration

Cargo.toml

@@ -225,7 +225,7 @@ ipnetwork = { version = "0.20", features = ["schemars"] }
 itertools = "0.11.0"
 key-manager = { path = "key-manager" }
 lazy_static = "1.4.0"
-libc = "0.2.149"
+libc = "0.2.150"
 linear-map = "1.2.0"
 macaddr = { version = "1.0.1", features = ["serde_std"] }
 mime_guess = "2.0.4"
@@ -365,7 +365,7 @@ tokio-util = "0.7.10"
 toml = "0.8.6"
 toml_edit = "0.20.7"
 topological-sort = "0.2.2"
-tough = { version = "0.12", features = [ "http" ] }
+tough = { version = "0.14", features = [ "http" ] }
 trust-dns-client = "0.22"
 trust-dns-proto = "0.22"
 trust-dns-resolver = "0.22"
@@ -376,14 +376,14 @@ tufaceous-lib = { path = "tufaceous-lib" }
 unicode-width = "0.1.11"
 update-engine = { path = "update-engine" }
 usdt = "0.3"
-uuid = { version = "1.4.1", features = ["serde", "v4"] }
+uuid = { version = "1.5.0", features = ["serde", "v4"] }
 walkdir = "2.4"
 wicket = { path = "wicket" }
 wicket-common = { path = "wicket-common" }
 wicketd-client = { path = "clients/wicketd-client" }
 zeroize = { version = "1.6.0", features = ["zeroize_derive", "std"] }
 zip = { version = "0.6.6", default-features = false, features = ["deflate","bzip2"] }
-zone = { version = "0.2", default-features = false, features = ["async"] }
+zone = { version = "0.3", default-features = false, features = ["async"] }
 
 [profile.dev]
 panic = "abort"

common/src/lib.rs

@@ -75,3 +75,5 @@ impl slog::KV for FileKv {
         )
     }
 }
+
+pub const OMICRON_DPD_TAG: &str = "omicron";

illumos-utils/src/running_zone.rs

@@ -517,12 +517,12 @@ impl RunningZone {
         // services are up, so future requests to create network addresses
         // or manipulate services will work.
         let fmri = "svc:/milestone/single-user:default";
-        wait_for_service(Some(&zone.name), fmri).await.map_err(|_| {
-            BootError::Timeout {
+        wait_for_service(Some(&zone.name), fmri, zone.log.clone())
+            .await
+            .map_err(|_| BootError::Timeout {
                 service: fmri.to_string(),
                 zone: zone.name.to_string(),
-            }
-        })?;
+            })?;
 
         // If the zone is self-assembling, then SMF service(s) inside the zone
         // will be creating the listen address for the zone's service(s),

illumos-utils/src/svc.rs

@@ -12,6 +12,7 @@ use omicron_common::backoff;
 #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))]
 mod inner {
     use super::*;
+    use slog::{warn, Logger};
 
     // TODO(https://www.illumos.org/issues/13837): This is a hack;
     // remove me when when fixed. Ideally, the ".synchronous()" argument
@@ -27,10 +28,19 @@ mod inner {
     pub async fn wait_for_service<'a, 'b>(
         zone: Option<&'a str>,
         fmri: &'b str,
+        log: Logger,
     ) -> Result<(), Error> {
         let name = smf::PropertyName::new("restarter", "state").unwrap();
 
-        let log_notification_failure = |_error, _delay| {};
+        let log_notification_failure = |error, delay| {
+            warn!(
+                log,
+                "wait for service {:?} failed: {}. retry in {:?}",
+                zone,
+                error,
+                delay
+            );
+        };
         backoff::retry_notify(
             backoff::retry_policy_local(),
             || async {
@@ -47,6 +57,26 @@ mod inner {
                         == &smf::PropertyValue::Astring("online".to_string())
                     {
                         return Ok(());
+                    } else {
+                        // This is helpful in virtual environments where
+                        // services take a few tries to come up. To enable,
+                        // compile with RUSTFLAGS="--cfg svcadm_autoclear"
+                        #[cfg(svcadm_autoclear)]
+                        if let Some(zname) = zone {
+                            if let Err(out) =
+                                tokio::process::Command::new(crate::PFEXEC)
+                                    .env_clear()
+                                    .arg("svcadm")
+                                    .arg("-z")
+                                    .arg(zname)
+                                    .arg("clear")
+                                    .arg("*")
+                                    .output()
+                                    .await
+                            {
+                                warn!(log, "clearing service maintenance failed: {out}");
+                            };
+                        }
                     }
                 }
                 return Err(backoff::BackoffError::transient(

nexus/src/app/sagas/mod.rs

@@ -313,7 +313,7 @@ macro_rules! declare_saga_actions {
     };
 }
 
-pub(crate) const NEXUS_DPD_TAG: &str = "nexus";
+use omicron_common::OMICRON_DPD_TAG as NEXUS_DPD_TAG;
 
 pub(crate) use __action_name;
 pub(crate) use __emit_action;

nexus/src/app/sagas/switch_port_settings_apply.rs

@@ -3,6 +3,7 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use super::{NexusActionContext, NEXUS_DPD_TAG};
+use crate::app::map_switch_zone_addrs;
 use crate::app::sagas::retry_until_known_result;
 use crate::app::sagas::{
     declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError,
@@ -15,15 +16,17 @@ use dpd_client::types::{
     RouteSettingsV4, RouteSettingsV6,
 };
 use dpd_client::{Ipv4Cidr, Ipv6Cidr};
+use internal_dns::ServiceName;
 use ipnetwork::IpNetwork;
 use mg_admin_client::types::Prefix4;
-use mg_admin_client::types::{ApplyRequest, BgpPeerConfig, BgpRoute};
+use mg_admin_client::types::{ApplyRequest, BgpPeerConfig};
 use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed, NETWORK_KEY};
 use nexus_db_queries::context::OpContext;
 use nexus_db_queries::db::datastore::UpdatePrecondition;
 use nexus_db_queries::{authn, db};
 use nexus_types::external_api::params;
-use omicron_common::api::external::{self, DataPageParams, NameOrId};
+use omicron_common::address::SLED_AGENT_PORT;
+use omicron_common::api::external::{self, NameOrId};
 use omicron_common::api::internal::shared::{
     ParseSwitchLocationError, SwitchLocation,
 };
@@ -35,8 +38,8 @@ use sled_agent_client::types::{
     BgpPeerConfig as OmicronBgpPeerConfig, HostPortConfig,
 };
 use std::collections::HashMap;
-use std::net::IpAddr;
 use std::net::SocketAddrV6;
+use std::net::{IpAddr, Ipv6Addr};
 use std::str::FromStr;
 use std::sync::Arc;
 use steno::ActionError;
@@ -177,7 +180,6 @@ pub(crate) fn api_to_dpd_port_settings(
     settings: &SwitchPortSettingsCombinedResult,
 ) -> Result<PortSettings, String> {
     let mut dpd_port_settings = PortSettings {
-        tag: NEXUS_DPD_TAG.into(),
         links: HashMap::new(),
         v4_routes: HashMap::new(),
         v6_routes: HashMap::new(),
@@ -192,6 +194,7 @@ pub(crate) fn api_to_dpd_port_settings(
             LinkSettings {
                 params: LinkCreate {
                     autoneg: false,
+                    lane: Some(LinkId(0)),
                     kr: false,
                     fec: match l.fec {
                         SwitchLinkFec::Firecode => PortFec::Firecode,
@@ -283,7 +286,13 @@ async fn spa_ensure_switch_port_settings(
         })?;
 
     retry_until_known_result(log, || async {
-        dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await
+        dpd_client
+            .port_settings_apply(
+                &port_id,
+                Some(NEXUS_DPD_TAG),
+                &dpd_port_settings,
+            )
+            .await
     })
     .await
     .map_err(|e| match e {
@@ -331,7 +340,9 @@ async fn spa_undo_ensure_switch_port_settings(
         Some(id) => id,
         None => {
             retry_until_known_result(log, || async {
-                dpd_client.port_settings_clear(&port_id).await
+                dpd_client
+                    .port_settings_clear(&port_id, Some(NEXUS_DPD_TAG))
+                    .await
             })
             .await
             .map_err(|e| external::Error::internal_error(&e.to_string()))?;
@@ -355,7 +366,13 @@ async fn spa_undo_ensure_switch_port_settings(
         })?;
 
     retry_until_known_result(log, || async {
-        dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await
+        dpd_client
+            .port_settings_apply(
+                &port_id,
+                Some(NEXUS_DPD_TAG),
+                &dpd_port_settings,
+            )
+            .await
     })
     .await
     .map_err(|e| external::Error::internal_error(&e.to_string()))?;
@@ -418,22 +435,6 @@ pub(crate) async fn ensure_switch_port_bgp_settings(
                 ))
             })?;
 
-        // TODO picking the first configured address by default, but this needs
-        // to be something that can be specified in the API.
-        let nexthop = match settings.addresses.get(0) {
-            Some(switch_port_addr) => Ok(switch_port_addr.address.ip()),
-            None => Err(ActionError::action_failed(
-                "at least one address required for bgp peering".to_string(),
-            )),
-        }?;
-
-        let nexthop = match nexthop {
-            IpAddr::V4(nexthop) => Ok(nexthop),
-            IpAddr::V6(_) => Err(ActionError::action_failed(
-                "IPv6 nexthop not yet supported".to_string(),
-            )),
-        }?;
-
         let mut prefixes = Vec::new();
         for a in &announcements {
             let value = match a.network.ip() {
@@ -455,7 +456,7 @@ pub(crate) async fn ensure_switch_port_bgp_settings(
             connect_retry: peer.connect_retry.0.into(),
             keepalive: peer.keepalive.0.into(),
             resolution: BGP_SESSION_RESOLUTION,
-            routes: vec![BgpRoute { nexthop, prefixes }],
+            originate: prefixes,
         };
 
         bgp_peer_configs.push(bpc);
@@ -809,7 +810,7 @@ pub(crate) async fn select_mg_client(
 }
 
 pub(crate) async fn get_scrimlet_address(
-    _location: SwitchLocation,
+    location: SwitchLocation,
     nexus: &Arc<Nexus>,
 ) -> Result<SocketAddrV6, ActionError> {
     /* TODO this depends on DNS entries only coming from RSS, it's broken
@@ -826,21 +827,41 @@ pub(crate) async fn get_scrimlet_address(
             ))
         })
     */
-    let opctx = &nexus.opctx_for_internal_api();
-    Ok(nexus
-        .sled_list(opctx, &DataPageParams::max_page())
+    let result = nexus
+        .resolver()
+        .await
+        .lookup_all_ipv6(ServiceName::Dendrite)
         .await
         .map_err(|e| {
             ActionError::action_failed(format!(
-                "get_scrimlet_address: failed to list sleds: {e}"
+                "scrimlet dns lookup failed {e}",
             ))
-        })?
-        .into_iter()
-        .find(|x| x.is_scrimlet())
-        .ok_or(ActionError::action_failed(
-            "get_scrimlet_address: no scrimlets found".to_string(),
-        ))?
-        .address())
+        });
+
+    let mappings = match result {
+        Ok(addrs) => map_switch_zone_addrs(&nexus.log, addrs).await,
+        Err(e) => {
+            warn!(nexus.log, "Failed to lookup Dendrite address: {e}");
+            return Err(ActionError::action_failed(format!(
+                "switch mapping failed {e}",
+            )));
+        }
+    };
+
+    let addr = match mappings.get(&location) {
+        Some(addr) => addr,
+        None => {
+            return Err(ActionError::action_failed(format!(
+                "address for switch at location: {location} not found",
+            )));
+        }
+    };
+
+    let mut segments = addr.segments();
+    segments[7] = 1;
+    let addr = Ipv6Addr::from(segments);
+
+    Ok(SocketAddrV6::new(addr, SLED_AGENT_PORT, 0, 0))
 }
 
 #[derive(Clone, Debug)]