From d0a1123ece1514ee5b65591b43794a5a059095fe Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 22 Apr 2024 15:51:46 +1200 Subject: [PATCH 01/56] Update the zone setup CLI to take several static addresses during common nw setup --- sled-agent/src/services.rs | 83 ++++++++++++++++++++++++-------- zone-setup/src/bin/zone-setup.rs | 46 ++++++++++++------ 2 files changed, 93 insertions(+), 36 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index e2e86e327e..8efd2e7409 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1421,7 +1421,7 @@ impl ServiceManager { fn zone_network_setup_install( gw_addr: &Ipv6Addr, zone: &InstalledZone, - static_addr: &String, + static_addrs: &Vec, ) -> Result { let datalink = zone.get_control_vnic_name(); let gateway = &gw_addr.to_string(); @@ -1429,8 +1429,15 @@ impl ServiceManager { let mut config_builder = PropertyGroupBuilder::new("config"); config_builder = config_builder .add_property("datalink", "astring", datalink) - .add_property("gateway", "astring", gateway) - .add_property("static_addr", "astring", static_addr); + .add_property("gateway", "astring", gateway); + + for s in static_addrs { + config_builder = config_builder.add_property( + "static_addr", + "astring", + &s.to_string(), + ); + } Ok(ServiceBuilder::new("oxide/zone-network-setup") .add_property_group(config_builder) @@ -1597,7 +1604,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - listen_addr, + &vec![*underlay_address], )?; let dns_service = Self::dns_install(info, None, &None).await?; @@ -1646,7 +1653,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - listen_addr, + &vec![*underlay_address], )?; let dns_service = Self::dns_install(info, None, &None).await?; @@ -1702,7 +1709,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - listen_addr, + &vec![*underlay_address], )?; let dns_service = Self::dns_install(info, None, &None).await?; @@ -1751,7 +1758,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - listen_addr, + &vec![*underlay_address], )?; let dataset_name = DatasetName::new( @@ -1806,7 +1813,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - listen_addr, + &vec![*underlay_address], )?; let config = PropertyGroupBuilder::new("config") @@ -1851,12 +1858,10 @@ impl ServiceManager { OXIMETER_PORT, ); - let listen_addr = &address.ip().to_string(); - let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - listen_addr, + &vec![*underlay_address], )?; let oximeter_config = PropertyGroupBuilder::new("config") @@ -1899,7 +1904,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &static_addr.clone(), + &vec![*underlay_address], )?; // Like Nexus, we need to be reachable externally via @@ -1983,12 +1988,10 @@ impl ServiceManager { return Err(Error::SledAgentNotReady); }; - let static_addr = underlay_address.to_string(); - let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &static_addr.clone(), + &vec![*underlay_address], )?; let is_boundary = matches!( @@ -2081,7 +2084,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( gz_address, &installed_zone, - &underlay_address.to_string(), + &vec![*underlay_address], )?; // Internal DNS zones require a special route through @@ -2161,12 +2164,10 @@ impl ServiceManager { return Err(Error::SledAgentNotReady); }; - let static_addr = underlay_address.to_string(); - let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &static_addr.clone(), + &vec![*underlay_address], )?; // While Nexus will be reachable via `external_ip`, it @@ -2289,7 +2290,49 @@ impl ServiceManager { })?; return Ok(RunningZone::boot(installed_zone).await?); } - _ => {} + _ => {} // ZoneArgs::Switch(SwitchZoneConfigLocal { + // zone: + // SwitchZoneConfig { + // id, + // services, + // addresses, + // .. + // }, + // .. + // }) => { + // let Some(info) = self.inner.sled_info.get() else { + // return Err(Error::SledAgentNotReady); + // }; + // + // // let static_addr = addresses; + // + // let nw_setup_service = Self::zone_network_setup_install( + // &info.underlay_address, + // &installed_zone, + // addresses, + // )?; + // + // let oximeter_config = PropertyGroupBuilder::new("config") + // .add_property("id", "astring", &id.to_string()); + // let oximeter_service = ServiceBuilder::new("oxide/oximeter") + // .add_instance( + // ServiceInstanceBuilder::new("default") + // .add_property_group(oximeter_config), + // ); + // + // let profile = ProfileBuilder::new("omicron") + // .add_service(nw_setup_service) + // .add_service(disabled_ssh_service) + // .add_service(oximeter_service) + // .add_service(disabled_dns_client_service); + // profile + // .add_to_zone(&self.inner.log, &installed_zone) + // .await + // .map_err(|err| { + // Error::io("Failed to setup Oximeter profile", err) + // })?; + // return Ok(RunningZone::boot(installed_zone).await?); + // } } let running_zone = RunningZone::boot(installed_zone).await?; diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 333f479721..59854a0067 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -113,11 +113,14 @@ async fn do_run() -> Result<(), CmdError> { .value_parser(parse_ipv6), ) .arg( - arg!( - -s --static_addr "static_addr" - ) + Arg::new("static_addrs") + .short('s') + .long("static_addrs") + .num_args(1..) + .value_delimiter(' ') + .value_parser(parse_ipv6) + .help("List of static addresses separated by a space") .required(true) - .value_parser(parse_ipv6), ), ) .subcommand( @@ -417,7 +420,10 @@ async fn common_nw_set_up( log: Logger, ) -> Result<(), CmdError> { let datalink: &String = matches.get_one("datalink").unwrap(); - let static_addr: &Ipv6Addr = matches.get_one("static_addr").unwrap(); + let static_addrs = matches + .get_many::("static_addrs") + .unwrap() + .collect::>(); let gateway: Ipv6Addr = *matches.get_one("gateway").unwrap(); let zonename = zone::current().await.map_err(|err| { CmdError::Failure(anyhow!( @@ -436,26 +442,34 @@ async fn common_nw_set_up( Ipadm::set_interface_mtu(&datalink) .map_err(|err| CmdError::Failure(anyhow!(err)))?; - info!(&log, "Ensuring static and auto-configured addresses are set on the IP interface"; "data link" => ?datalink, "static address" => ?static_addr); - Ipadm::create_static_and_autoconfigured_addrs(&datalink, static_addr) - .map_err(|err| CmdError::Failure(anyhow!(err)))?; + for addr in &static_addrs { + info!(&log, "Ensuring static and auto-configured addresses are set on the IP interface"; "data link" => ?datalink, "static address" => ?addr); + Ipadm::create_static_and_autoconfigured_addrs(&datalink, addr) + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + } info!(&log, "Ensuring there is a default route"; "gateway" => ?gateway); Route::ensure_default_route_with_gateway(Gateway::Ipv6(gateway)) .map_err(|err| CmdError::Failure(anyhow!(err)))?; info!(&log, "Populating hosts file for zone"; "zonename" => ?zonename); - write( - HOSTS_FILE, - format!( - r#" + let mut hosts_contents = String::from( + r#" ::1 localhost loghost 127.0.0.1 localhost loghost -{static_addr} {zonename}.local {zonename} +"#, + ); + + for addr in static_addrs.clone() { + let s = format!( + r#"{addr} {zonename}.local {zonename} "# - ), - ) - .map_err(|err| CmdError::Failure(anyhow!(err)))?; + ); + hosts_contents.push_str(s.as_str()) + } + + write(HOSTS_FILE, hosts_contents) + .map_err(|err| CmdError::Failure(anyhow!(err)))?; Ok(()) } From 444e9fb493f47d5c0fa90a4b03d7ae892450233f Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 22 Apr 2024 17:31:16 +1200 Subject: [PATCH 02/56] Set MGS service --- sled-agent/src/services.rs | 353 +++++++++++++++++++++---------------- smf/mgs/manifest.xml | 2 +- 2 files changed, 206 insertions(+), 149 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 8efd2e7409..6f8a3ee8b5 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -2290,156 +2290,213 @@ impl ServiceManager { })?; return Ok(RunningZone::boot(installed_zone).await?); } - _ => {} // ZoneArgs::Switch(SwitchZoneConfigLocal { - // zone: - // SwitchZoneConfig { - // id, - // services, - // addresses, - // .. - // }, - // .. - // }) => { - // let Some(info) = self.inner.sled_info.get() else { - // return Err(Error::SledAgentNotReady); - // }; - // - // // let static_addr = addresses; - // - // let nw_setup_service = Self::zone_network_setup_install( - // &info.underlay_address, - // &installed_zone, - // addresses, - // )?; - // - // let oximeter_config = PropertyGroupBuilder::new("config") - // .add_property("id", "astring", &id.to_string()); - // let oximeter_service = ServiceBuilder::new("oxide/oximeter") - // .add_instance( - // ServiceInstanceBuilder::new("default") - // .add_property_group(oximeter_config), - // ); - // - // let profile = ProfileBuilder::new("omicron") - // .add_service(nw_setup_service) - // .add_service(disabled_ssh_service) - // .add_service(oximeter_service) - // .add_service(disabled_dns_client_service); - // profile - // .add_to_zone(&self.inner.log, &installed_zone) - // .await - // .map_err(|err| { - // Error::io("Failed to setup Oximeter profile", err) - // })?; - // return Ok(RunningZone::boot(installed_zone).await?); - // } - } - - let running_zone = RunningZone::boot(installed_zone).await?; + ZoneArgs::Switch(SwitchZoneConfigLocal { + zone: SwitchZoneConfig { id, services, addresses, .. }, + .. + }) => { + let Some(info) = self.inner.sled_info.get() else { + return Err(Error::SledAgentNotReady); + }; - for (link, needs_link_local) in - running_zone.links().iter().zip(links_need_link_local) - { - if needs_link_local { - info!( - self.inner.log, - "Ensuring {}/{} exists in zone", - link.name(), - IPV6_LINK_LOCAL_NAME - ); - Zones::ensure_has_link_local_v6_address( - Some(running_zone.name()), - &AddrObject::new(link.name(), IPV6_LINK_LOCAL_NAME) - .unwrap(), + let nw_setup_service = Self::zone_network_setup_install( + &info.underlay_address, + &installed_zone, + addresses, )?; - } - } - if let Some((bootstrap_name, bootstrap_address)) = - bootstrap_name_and_address.as_ref() - { - info!( - self.inner.log, - "Ensuring bootstrap address {} exists in {} zone", - bootstrap_address.to_string(), - &zone_type_str, - ); - running_zone.ensure_bootstrap_address(*bootstrap_address).await?; - info!( - self.inner.log, - "Forwarding bootstrap traffic via {} to {}", - bootstrap_name, - self.inner.global_zone_bootstrap_link_local_address, - ); - running_zone - .add_bootstrap_route( - BOOTSTRAP_PREFIX, - self.inner.global_zone_bootstrap_link_local_address, - bootstrap_name, - ) - .map_err(|err| Error::ZoneCommand { - intent: "add bootstrap network route".to_string(), - err, - })?; - } + let sidecar_revision = match &self.inner.sidecar_revision { + SidecarRevision::Physical(rev) => rev.to_string(), + SidecarRevision::SoftZone(rev) + | SidecarRevision::SoftPropolis(rev) => format!( + "softnpu_front_{}_rear_{}", + rev.front_port_count, rev.rear_port_count + ), + }; - let addresses = match &request { - ZoneArgs::Omicron(OmicronZoneConfigLocal { - zone: OmicronZoneConfig { underlay_address, .. }, - .. - }) => std::slice::from_ref(underlay_address), - ZoneArgs::Switch(req) => &req.zone.addresses, - }; - for addr in addresses { - if *addr == Ipv6Addr::LOCALHOST { - continue; - } - info!( - self.inner.log, - "Ensuring address {} exists", - addr.to_string() - ); - let addr_request = - AddressRequest::new_static(IpAddr::V6(*addr), None); - running_zone.ensure_address(addr_request).await?; - info!( - self.inner.log, - "Ensuring address {} exists - OK", - addr.to_string() - ); - } + // Define all services in the switch zone + let mut mgs_service = ServiceBuilder::new("oxide/mgs"); - let maybe_gateway = if let Some(info) = self.inner.sled_info.get() { - // Only consider a route to the sled's underlay address if the - // underlay is up. - let sled_underlay_subnet = - Ipv6Subnet::::new(info.underlay_address); + // Set properties for each service + for service in services { + match service { + SwitchService::ManagementGatewayService => { + let mut mgs_config = + PropertyGroupBuilder::new("config") + // Always tell MGS to listen on localhost so wicketd + // can contact it even before we have an underlay + // network. + .add_property( + "address", + "astring", + &format!("[::1]:{MGS_PORT}"), + ) + .add_property( + "id", + "astring", + &id.to_string(), + ) + .add_property( + "rack_id", + "astring", + &info.rack_id.to_string(), + ); - if addresses - .iter() - .any(|ip| sled_underlay_subnet.net().contains(*ip)) - { - // If the underlay is up, provide a route to it through an - // existing address in the Zone on the same subnet. - info!(self.inner.log, "Zone using sled underlay as gateway"); - Some(info.underlay_address) - } else { - // If no such address exists in the sled's subnet, don't route - // to anything. - info!( - self.inner.log, - "Zone not using gateway (even though underlay is up)" - ); - None + if let Some(address) = addresses.get(0) { + // Don't use localhost twice + if *address != Ipv6Addr::LOCALHOST { + mgs_config = mgs_config.add_property( + "address", + "astring", + &format!("[{address}]:{MGS_PORT}"), + ); + } + } + mgs_service = mgs_service.add_instance( + ServiceInstanceBuilder::new("default") + .add_property_group(mgs_config), + ); + } + SwitchService::SpSim => {} + SwitchService::Wicketd { baseboard } => {} + SwitchService::Dendrite { asic } => match asic { + DendriteAsic::TofinoAsic => {} + DendriteAsic::TofinoStub => {} + asic @ (DendriteAsic::SoftNpuZone + | DendriteAsic::SoftNpuPropolisDevice) => {} + }, + SwitchService::Tfport { pkt_source, asic } => {} + SwitchService::Lldpd { baseboard } => {} + SwitchService::Uplink => { + // Nothing to do here - this service is special and + // configured in + // `ensure_switch_zone_uplinks_configured` + } + SwitchService::Mgd => {} + SwitchService::MgDdm { mode } => {} + } + } + + let profile = ProfileBuilder::new("omicron") + .add_service(nw_setup_service) + .add_service(disabled_ssh_service) + .add_service(mgs_service) + .add_service(disabled_dns_client_service); + profile + .add_to_zone(&self.inner.log, &installed_zone) + .await + .map_err(|err| { + Error::io("Failed to setup Oximeter profile", err) + })?; + return Ok(RunningZone::boot(installed_zone).await?); } - } else { - // If the underlay doesn't exist, no routing occurs. - info!( - self.inner.log, - "Zone not using gateway (underlay is not up)" - ); - None - }; + } + + // TODO: Remove from here until end + let running_zone = RunningZone::boot(installed_zone).await?; + // + // for (link, needs_link_local) in + // running_zone.links().iter().zip(links_need_link_local) + // { + // if needs_link_local { + // info!( + // self.inner.log, + // "Ensuring {}/{} exists in zone", + // link.name(), + // IPV6_LINK_LOCAL_NAME + // ); + // Zones::ensure_has_link_local_v6_address( + // Some(running_zone.name()), + // &AddrObject::new(link.name(), IPV6_LINK_LOCAL_NAME) + // .unwrap(), + // )?; + // } + // } + + // if let Some((bootstrap_name, bootstrap_address)) = + // bootstrap_name_and_address.as_ref() + // { + // info!( + // self.inner.log, + // "Ensuring bootstrap address {} exists in {} zone", + // bootstrap_address.to_string(), + // &zone_type_str, + // ); + // running_zone.ensure_bootstrap_address(*bootstrap_address).await?; + // info!( + // self.inner.log, + // "Forwarding bootstrap traffic via {} to {}", + // bootstrap_name, + // self.inner.global_zone_bootstrap_link_local_address, + // ); + // running_zone + // .add_bootstrap_route( + // BOOTSTRAP_PREFIX, + // self.inner.global_zone_bootstrap_link_local_address, + // bootstrap_name, + // ) + // .map_err(|err| Error::ZoneCommand { + // intent: "add bootstrap network route".to_string(), + // err, + // })?; + // } + + // let addresses = match &request { + // ZoneArgs::Omicron(OmicronZoneConfigLocal { + // zone: OmicronZoneConfig { underlay_address, .. }, + // .. + // }) => std::slice::from_ref(underlay_address), + // ZoneArgs::Switch(req) => &req.zone.addresses, + // }; + // for addr in addresses { + // if *addr == Ipv6Addr::LOCALHOST { + // continue; + // } + // info!( + // self.inner.log, + // "Ensuring address {} exists", + // addr.to_string() + // ); + // let addr_request = + // AddressRequest::new_static(IpAddr::V6(*addr), None); + // running_zone.ensure_address(addr_request).await?; + // info!( + // self.inner.log, + // "Ensuring address {} exists - OK", + // addr.to_string() + // ); + // } + + // let maybe_gateway = if let Some(info) = self.inner.sled_info.get() { + // // Only consider a route to the sled's underlay address if the + // // underlay is up. + // let sled_underlay_subnet = + // Ipv6Subnet::::new(info.underlay_address); + // + // if addresses + // .iter() + // .any(|ip| sled_underlay_subnet.net().contains(*ip)) + // { + // // If the underlay is up, provide a route to it through an + // // existing address in the Zone on the same subnet. + // info!(self.inner.log, "Zone using sled underlay as gateway"); + // Some(info.underlay_address) + // } else { + // // If no such address exists in the sled's subnet, don't route + // // to anything. + // info!( + // self.inner.log, + // "Zone not using gateway (even though underlay is up)" + // ); + // None + // } + // } else { + // // If the underlay doesn't exist, no routing occurs. + // info!( + // self.inner.log, + // "Zone not using gateway (underlay is not up)" + // ); + // None + // }; let sidecar_revision = match &self.inner.sidecar_revision { SidecarRevision::Physical(rev) => rev.to_string(), @@ -2450,11 +2507,11 @@ impl ServiceManager { ), }; - if let Some(gateway) = maybe_gateway { - running_zone.add_default_route(gateway).map_err(|err| { - Error::ZoneCommand { intent: "Adding Route".to_string(), err } - })?; - } + // if let Some(gateway) = maybe_gateway { + // running_zone.add_default_route(gateway).map_err(|err| { + // Error::ZoneCommand { intent: "Adding Route".to_string(), err } + // })?; + // } match &request { ZoneArgs::Omicron(zone_config) => { diff --git a/smf/mgs/manifest.xml b/smf/mgs/manifest.xml index 125c32ce2b..ea99a60537 100644 --- a/smf/mgs/manifest.xml +++ b/smf/mgs/manifest.xml @@ -4,7 +4,7 @@ - + From b82297b2f730fab17777262e528fc6e80263f487 Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 22 Apr 2024 18:48:25 +1200 Subject: [PATCH 03/56] Set Wicket service (almost) --- sled-agent/src/services.rs | 91 ++++++++++++++++++++++++- smf/mgs/manifest.xml | 5 ++ smf/switch_zone_setup/manifest.xml | 6 ++ smf/switch_zone_setup/switch_zone_setup | 3 + smf/wicketd/baseboard.json | 0 smf/wicketd/manifest.xml | 17 ++++- zone-setup/src/bin/zone-setup.rs | 63 +++++++++++++++++ 7 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 smf/wicketd/baseboard.json diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 6f8a3ee8b5..d043c81be6 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -2291,7 +2291,7 @@ impl ServiceManager { return Ok(RunningZone::boot(installed_zone).await?); } ZoneArgs::Switch(SwitchZoneConfigLocal { - zone: SwitchZoneConfig { id, services, addresses, .. }, + zone: SwitchZoneConfig { id, services, addresses}, .. }) => { let Some(info) = self.inner.sled_info.get() else { @@ -2315,6 +2315,7 @@ impl ServiceManager { // Define all services in the switch zone let mut mgs_service = ServiceBuilder::new("oxide/mgs"); + let mut wicketd_service = ServiceBuilder::new("oxide/wicketd"); // Set properties for each service for service in services { @@ -2356,8 +2357,92 @@ impl ServiceManager { .add_property_group(mgs_config), ); } - SwitchService::SpSim => {} - SwitchService::Wicketd { baseboard } => {} + SwitchService::SpSim => { + info!( + self.inner.log, + "Setting up Simulated SP service" + ); + } + SwitchService::Wicketd { baseboard } => { + // If we're launching the switch zone, we'll have a + // bootstrap_address based on our call to + // `self.bootstrap_address_needed` (which always + // gives us an address for the switch zone. If we + // _don't_ have a bootstrap address, someone has + // requested wicketd in a non-switch zone; return an + // error. + let Some((_, bootstrap_address)) = + bootstrap_name_and_address + else { + return Err(Error::BadServiceRequest { + service: "wicketd".to_string(), + message: concat!( + "missing bootstrap address: ", + "wicketd can only be started in the ", + "switch zone", + ) + .to_string(), + }); + }; + + let rack_subnet = Ipv6Subnet::::new( + info.underlay_address, + ); + + let baseboard_info = serde_json::to_string_pretty(&baseboard)?; + + let wicketd_config = + PropertyGroupBuilder::new("config") + .add_property( + "address", + "astring", + &format!("[::1]:{WICKETD_PORT}"), + ) + .add_property( + "artifact-address", + "astring", + &format!("[{bootstrap_address}]:{BOOTSTRAP_ARTIFACT_PORT}"), + ) + .add_property( + "mgs-address", + "astring", + &format!("[::1]:{MGS_PORT}"), + ) + // We intentionally bind `nexus-proxy-address` to + // `::` so wicketd will serve this on all + // interfaces, particularly the tech port + // interfaces, allowing external clients to connect + // to this Nexus proxy. + .add_property( + "nexus-proxy-address", + "astring", + &format!("[::]:{WICKETD_NEXUS_PROXY_PORT}"), + ) + .add_property( + "rack-subnet", + "astring", + &rack_subnet.net().ip().to_string(), + ) + .add_property( + "config/baseboard-file", + "astring", + // TODO: Used to be "/opt/oxide/baseboard.json", make sure + // it's OK to change + "/var/svc/manifest/site/wicketd/baseboard.json", + ) + // TODO: Remove this baseboard info and send it to + // switch_zone_setup service instead? + .add_property( + "config/baseboard-info", + "astring", + &baseboard_info, + ); + + wicketd_service = wicketd_service.add_instance( + ServiceInstanceBuilder::new("default") + .add_property_group(wicketd_config), + ); + } SwitchService::Dendrite { asic } => match asic { DendriteAsic::TofinoAsic => {} DendriteAsic::TofinoStub => {} diff --git a/smf/mgs/manifest.xml b/smf/mgs/manifest.xml index ea99a60537..e129ccf35a 100644 --- a/smf/mgs/manifest.xml +++ b/smf/mgs/manifest.xml @@ -11,6 +11,11 @@ + + + + diff --git a/smf/switch_zone_setup/switch_zone_setup b/smf/switch_zone_setup/switch_zone_setup index ad12677866..62160f1849 100755 --- a/smf/switch_zone_setup/switch_zone_setup +++ b/smf/switch_zone_setup/switch_zone_setup @@ -46,4 +46,7 @@ for i in "${!USERS[@]}"; do fi done +# TODO: Call /opt/oxide/zone-setup-cli/bin/zone-setup wicket-setup -b BASEBOARD_INFO here +# Eventually we'll want all of the above code to be part of the zone-setup CLI as well + exit $SMF_EXIT_OK diff --git a/smf/wicketd/baseboard.json b/smf/wicketd/baseboard.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/smf/wicketd/manifest.xml b/smf/wicketd/manifest.xml index b45ff1544b..1bf0c21371 100644 --- a/smf/wicketd/manifest.xml +++ b/smf/wicketd/manifest.xml @@ -4,13 +4,23 @@ - + + + + + + + + + @@ -45,6 +55,11 @@ + + + diff --git a/smf/switch_zone_setup/switch_zone_setup b/smf/switch_zone_setup/switch_zone_setup index 62160f1849..62755ad3ad 100755 --- a/smf/switch_zone_setup/switch_zone_setup +++ b/smf/switch_zone_setup/switch_zone_setup @@ -4,6 +4,8 @@ set -ex -o pipefail . /lib/svc/share/smf_include.sh +BASEBOARD_INFO="$(svcprop -c -p config/baseboard-info "${SMF_FMRI}")" + # set up the users required for wicket and support. USERS=( (user=wicket group=wicket gecos='Wicket User' nopasswd=1 shell='/bin/sh') @@ -46,7 +48,7 @@ for i in "${!USERS[@]}"; do fi done -# TODO: Call /opt/oxide/zone-setup-cli/bin/zone-setup wicket-setup -b BASEBOARD_INFO here -# Eventually we'll want all of the above code to be part of the zone-setup CLI as well +# TODO: Eventually we'll want all of the above code to be part of the zone-setup CLI as well +exec /opt/oxide/zone-setup-cli/bin/zone-setup switch-zone -i "$BASEBOARD_INFO" exit $SMF_EXIT_OK diff --git a/smf/wicketd/baseboard.json b/smf/wicketd/baseboard.json deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/smf/wicketd/manifest.xml b/smf/wicketd/manifest.xml index 1bf0c21371..a1f7a63b6c 100644 --- a/smf/wicketd/manifest.xml +++ b/smf/wicketd/manifest.xml @@ -55,12 +55,6 @@ - - - diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 75d3a159a6..7b181d3a98 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -438,38 +438,6 @@ async fn do_run() -> Result<(), CmdError> { .required(true) .value_parser(parse_baseboard_info), ) - .arg( - arg!( - -a --bootstrap_addr "bootstrap_addr" - ) - .required(true) - .value_parser(parse_ipv6), - ) - .arg( - arg!( - -n --bootstrap_name "bootstrap_name" - ) - .required(true), - ) - .arg( - arg!( - -v --bootstrap_vnic "bootstrap_vnic" - ) - .required(true), - ) - .arg( - arg!( - -g --gz_local_link_addr "gz_local_link_addr" - ) - .required(true) - .value_parser(parse_ipv6), - ) - .arg( - arg!( - -z --zone_name "zone_name" - ) - .required(true), - ) .arg( Arg::new("link_local_links") .short('l') @@ -535,13 +503,6 @@ async fn switch_zone_setup( ) -> Result<(), CmdError> { let file: &String = matches.get_one("baseboard_file").unwrap(); let info: &String = matches.get_one("baseboard_info").unwrap(); - // TODO: Get rid of zone name? - let zone_name: &String = matches.get_one("zone_name").unwrap(); - let bootstrap_addr: &Ipv6Addr = matches.get_one("bootstrap_addr").unwrap(); - let bootstrap_name: &String = matches.get_one("bootstrap_name").unwrap(); - let bootstrap_vnic: &String = matches.get_one("bootstrap_vnic").unwrap(); - let gz_local_link_addr: &Ipv6Addr = - matches.get_one("gz_local_link_addr").unwrap(); let links = if let Some(l) = matches.get_many::("link_local_links") { Some(l.collect::>()) @@ -579,10 +540,8 @@ async fn switch_zone_setup( } if let Some(links) = links { - info!(&log, "Ensuring link local links"; "links" => ?links, "zone name" => ?zone_name); + info!(&log, "Ensuring link local links"; "links" => ?links); for link in &links { - // println!("{link}"); - // println!("{zone_name}"); Zones::ensure_has_link_local_v6_address( None, &AddrObject::new(link, IPV6_LINK_LOCAL_NAME).unwrap(), @@ -599,48 +558,6 @@ async fn switch_zone_setup( info!(&log, "No link local links to be configured"); }; - // TODO: This CANNOT happen inside the zone as the vnics live in the global zone - // Oooooor can it???? - - info!(&log, "Ensuring bootstrap address exists in zone"; - "bootstrap address" => ?bootstrap_addr, "bootstrap vnic" => ?bootstrap_vnic, "bootstrap address" => ?bootstrap_addr); - // let addrtype = - // AddressRequest::new_static(std::net::IpAddr::V6(*bootstrap_addr), None); - // let addrobj_name = "bootstrap6"; - // let addrobj = - // AddrObject::new(&bootstrap_vnic, addrobj_name).map_err(|err| { - // CmdError::Failure(anyhow!( - // "Could not create new addrobj {:?}: {}", - // addrobj_name, - // err - // )) - // })?; - // // TODO: Fix error - // // root@oxz_switch:~# /usr/sbin/ipadm create-addr -t -T static -a fdb0:a8a1:59c7:4e85::2/64 oxBootstrap0/bootstrap6 - // // ipadm: Could not create address: Can't assign requested address - // let _ = Zones::ensure_address(None, &addrobj, addrtype) - // .map_err(|err| { - // CmdError::Failure(anyhow!( - // "Could not ensure address {} {:?}: {}", - // addrobj, - // addrtype, - // err - // )) - // })?; - // - info!( - &log, - "Forwarding bootstrap traffic via {} to {}", - bootstrap_name, - gz_local_link_addr - ); - // Route::add_bootstrap_route( - // BOOTSTRAP_PREFIX, - // *gz_local_link_addr, - // &bootstrap_name, - // ) - // .map_err(|err| CmdError::Failure(anyhow!(err)))?; - Ok(()) } From 0fcd764ca818b4f8f9acfd9a84dda7deb4565c3d Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 11 Jun 2024 16:31:46 +1200 Subject: [PATCH 31/56] Clean up switch zone setup --- zone-setup/src/bin/zone-setup.rs | 346 ++++++++++++++++++------------- 1 file changed, 200 insertions(+), 146 deletions(-) diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 7b181d3a98..7664a70c7d 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -55,26 +55,30 @@ impl SwitchZoneUser { group: String, gecos: String, nopasswd: bool, - homedir: Option, shell: String, - profiles: Option>, - ) -> SwitchZoneUser { - SwitchZoneUser { + ) -> Self { + Self { user, group, gecos, nopasswd, - homedir, + homedir: None, shell, - profiles, + profiles: None, } } - // TODO: Use builder pattern to set values + fn with_homedir(mut self, homedir: String) -> Self { + self.homedir = Some(homedir); + self + } - fn setup_switch_zone_user(self, log: &Logger) -> Result<(), CmdError> { - // TODO: Remove non-destructive comments I'm using for testing on my machine - info!(&log, "Add a new group for the user"; "group" => &self.group, "user" => &self.user); + fn with_profiles(mut self, profiles: Vec) -> Self { + self.profiles = Some(profiles); + self + } + + fn add_new_group_for_user(&self) -> Result<(), CmdError> { match get_group_by_name(&self.group) { Some(_) => {} None => { @@ -98,9 +102,10 @@ impl SwitchZoneUser { } } }; + Ok(()) + } - info!(&log, "Add the user"; "user" => &self.user, "shell" => &self.shell, - "group" => &self.group, "gecos" => &self.gecos); + fn add_new_user(&self) -> Result<(), CmdError> { match get_user_by_name(&self.user) { Some(_) => {} None => { @@ -132,150 +137,197 @@ impl SwitchZoneUser { } } }; + Ok(()) + } - // Either enable password-less login (wicket) or disable password-based logins - // completely (support, which logs in via ssh key). - if self.nopasswd { - info!(&log, "Enable password-less login for user"; "user" => self.user.clone()); - let cmd = std::process::Command::new("passwd") - .args(["-d", &self.user]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `passwd -d {}`: {}", - self.user, - err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not enable password-less login: {} status: {}", - self.user, - cmd.status - ))); - } - } else { - info!(&log, "Disable password-based logins"; "user" => self.user.clone()); - let cmd = std::process::Command::new("passwd") - .args(["-N", &self.user]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `passwd -N {}`: {}", - self.user, - err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not disable password-based logins: {} status: {}", + fn enable_passwordless_login(&self) -> Result<(), CmdError> { + let cmd = std::process::Command::new("passwd") + .args(["-d", &self.user]) + .output() + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute `passwd -d {}`: {}", self.user, - cmd.status - ))); - } - }; - - if let Some(profiles) = self.profiles { - let mut profile_list: String = Default::default(); - for profile in profiles { - info!(&log, "Assign user profiles"; "user" => &self.user, "profile" => &profile); - profile_list.push_str(&profile) - } + err + )) + })?; - let cmd = std::process::Command::new("usermod") - .args(["-P", &profile_list, &self.user]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `usermod -P {} {}`: {}", - profile_list, - self.user, - err - )) - })?; + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not enable password-less login: {} status: {}", + self.user, + cmd.status + ))); + } + Ok(()) + } - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not assign user profiles: {} status: {}", + fn disable_password_based_login(&self) -> Result<(), CmdError> { + let cmd = std::process::Command::new("passwd") + .args(["-N", &self.user]) + .output() + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute `passwd -N {}`: {}", self.user, - cmd.status - ))); - } - } else { - info!(&log, "Remove user profiles"; "user" => self.user.clone()); - let cmd = std::process::Command::new("usermod") - .args(["-P", "", &self.user]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `usermod -P '' {}`: {}", - self.user, - err - )) - })?; + err + )) + })?; - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not remove user profiles: {} status: {}", - self.user, - cmd.status - ))); - } + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not disable password-based logins: {} status: {}", + self.user, + cmd.status + ))); + } + Ok(()) + } + + fn assign_user_profiles(&self) -> Result<(), CmdError> { + let Some(ref profiles) = self.profiles else { + return Err(CmdError::Failure(anyhow!( + "Profile list must not be empty to assign user profiles", + ))); }; - if let Some(homedir) = self.homedir { - info!(&log, "Set up home directory and startup files"; "user" => self.user.clone(), "home directory" => homedir.clone()); - create_dir_all(&homedir).map_err(|err| { + let mut profile_list: String = Default::default(); + for profile in profiles { + profile_list.push_str(&profile) + } + + let cmd = std::process::Command::new("usermod") + .args(["-P", &profile_list, &self.user]) + .output() + .map_err(|err| { CmdError::Failure(anyhow!( - "Could not execute create directory {} and its parents: {}", - &homedir, + "Could not execute `usermod -P {} {}`: {}", + profile_list, + self.user, err )) })?; - let mut home_bashrc = homedir.clone(); - home_bashrc.push_str("/.bashrc"); - copy("/root/.bashrc", &home_bashrc).map_err(|err| { + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not assign user profiles: {} status: {}", + self.user, + cmd.status + ))); + } + Ok(()) + } + + fn remove_user_profiles(&self) -> Result<(), CmdError> { + let cmd = std::process::Command::new("usermod") + .args(["-P", "", &self.user]) + .output() + .map_err(|err| { CmdError::Failure(anyhow!( - "Could not copy file from /root/.bashrc to {}: {}", - &homedir, + "Could not execute `usermod -P '' {}`: {}", + self.user, err )) })?; - let mut home_profile = homedir.clone(); - home_profile.push_str("/.profile"); - copy("/root/.profile", &home_profile).map_err(|err| { + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not remove user profiles: {} status: {}", + self.user, + cmd.status + ))); + } + Ok(()) + } + + fn set_up_home_directory_and_startup_files(&self) -> Result<(), CmdError> { + let Some(ref homedir) = self.homedir else { + return Err(CmdError::Failure(anyhow!( + "A home directory must be provided to set up home directory and startup files", + ))); + }; + + create_dir_all(&homedir).map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute create directory {} and its parents: {}", + &homedir, + err + )) + })?; + + let mut home_bashrc = homedir.clone(); + home_bashrc.push_str("/.bashrc"); + copy("/root/.bashrc", &home_bashrc).map_err(|err| { + CmdError::Failure(anyhow!( + "Could not copy file from /root/.bashrc to {}: {}", + &homedir, + err + )) + })?; + + let mut home_profile = homedir.clone(); + home_profile.push_str("/.profile"); + copy("/root/.profile", &home_profile).map_err(|err| { + CmdError::Failure(anyhow!( + "Could not copy file from /root/.profile to {}: {}", + &homedir, + err + )) + })?; + + // Not using std::os::unix::fs::chown here because it doesn't support + // recursive option. + let cmd = std::process::Command::new("chown") + .args(["-R", &self.user, &homedir]) + .output() + .map_err(|err| { CmdError::Failure(anyhow!( - "Could not copy file from /root/.profile to {}: {}", - &homedir, + "Could not execute `chown -R {} {}`: {}", + self.user, + homedir, err )) })?; - // Not using std::os::unix::fs::chown here because it doesn't support - // recursive option. - let cmd = std::process::Command::new("chown") - .args(["-R", &self.user, &homedir]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `chown -R {} {}`: {}", - self.user, - homedir, - err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not change ownership: {} status: {}", - homedir, - cmd.status - ))); - } + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not change ownership: {} status: {}", + homedir, + cmd.status + ))); + } + Ok(()) + } + + fn setup_switch_zone_user(self, log: &Logger) -> Result<(), CmdError> { + info!(&log, "Add a new group for the user"; "group" => ?self.group, "user" => ?self.user); + self.add_new_group_for_user()?; + + info!(&log, "Add the user"; "user" => ?self.user, "shell" => ?self.shell, + "group" => &self.group, "gecos" => &self.gecos); + self.add_new_user()?; + + // Either enable password-less login (wicket) or disable password-based logins + // completely (support, which logs in via ssh key). + if self.nopasswd { + info!(&log, "Enable password-less login for user"; "user" => ?self.user); + self.enable_passwordless_login()?; + } else { + info!(&log, "Disable password-based logins"; "user" => ?self.user); + self.disable_password_based_login()?; + }; + + if let Some(_) = &self.profiles { + info!(&log, "Assign user profiles"; "user" => ?self.user, "profiles" => ?self.profiles); + self.assign_user_profiles()?; + } else { + info!(&log, "Remove user profiles"; "user" => ?self.user); + self.remove_user_profiles()?; + }; + + if let Some(_) = self.homedir { + info!(&log, "Set up home directory and startup files"; "user" => ?self.user, "home directory" => ?self.homedir); + self.set_up_home_directory_and_startup_files()?; } Ok(()) } @@ -515,24 +567,26 @@ async fn switch_zone_setup( info!(&log, "Setting up the users required for wicket and support"); let wicket_user = SwitchZoneUser::new( - String::from("wicket"), - String::from("wicket"), - String::from("Wicket User"), + "wicket".to_string(), + "wicket".to_string(), + "Wicket User".to_string(), true, - None, - String::from("/bin/sh"), - None, + // None, + "/bin/sh".to_string(), + // None, ); let support_user = SwitchZoneUser::new( - String::from("support"), - String::from("support"), - String::from("Oxide Support"), + "support".to_string(), + "support".to_string(), + "Oxide Support".to_string(), false, - Some(String::from("/home/support")), - String::from("/bin/bash"), - Some(vec![String::from("Primary Administrator")]), - ); + // Some(String::from("/home/support")), + "/bin/bash".to_string(), + // Some(vec![String::from("Primary Administrator")]), + ) + .with_homedir("/home/support".to_string()) + .with_profiles(vec!["Primary Administrator".to_string()]); let users = vec![wicket_user, support_user]; for u in users { From ab757341564902a7dadfb843536d2244ef9fd7ff Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 11 Jun 2024 17:48:38 +1200 Subject: [PATCH 32/56] extract switch zone user --- zone-setup/src/bin/zone-setup.rs | 302 +------------------------------ 1 file changed, 2 insertions(+), 300 deletions(-) diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 7664a70c7d..5df204688b 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -18,7 +18,7 @@ use omicron_sled_agent::services::SWITCH_ZONE_BASEBOARD_FILE; use serde_json::Value; use slog::{info, Logger}; use std::fs::{ - copy, create_dir_all, metadata, read_to_string, set_permissions, write, + metadata, read_to_string, set_permissions, write, OpenOptions, }; use std::io::Write; @@ -26,6 +26,7 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::os::unix::fs::chown; use std::path::Path; use uzers::{get_group_by_name, get_user_by_name}; +use zone_setup::switch_zone_user::SwitchZoneUser; pub const HOSTS_FILE: &str = "/etc/inet/hosts"; pub const CHRONY_CONFIG_FILE: &str = "/etc/inet/chrony.conf"; @@ -38,301 +39,6 @@ pub const OPTE_INTERFACE_CMD: &str = "opte-interface"; pub const CHRONY_SETUP_CMD: &str = "chrony-setup"; pub const SWITCH_ZONE_SETUP_CMD: &str = "switch-zone"; -// User information for the switch zone -struct SwitchZoneUser { - user: String, - group: String, - gecos: String, - nopasswd: bool, - homedir: Option, - shell: String, - profiles: Option>, -} - -impl SwitchZoneUser { - fn new( - user: String, - group: String, - gecos: String, - nopasswd: bool, - shell: String, - ) -> Self { - Self { - user, - group, - gecos, - nopasswd, - homedir: None, - shell, - profiles: None, - } - } - - fn with_homedir(mut self, homedir: String) -> Self { - self.homedir = Some(homedir); - self - } - - fn with_profiles(mut self, profiles: Vec) -> Self { - self.profiles = Some(profiles); - self - } - - fn add_new_group_for_user(&self) -> Result<(), CmdError> { - match get_group_by_name(&self.group) { - Some(_) => {} - None => { - let cmd = std::process::Command::new("groupadd") - .arg(&self.group) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `groupadd {}`: {}", - self.group, - err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not add group: {} status: {}", - self.group, - cmd.status - ))); - } - } - }; - Ok(()) - } - - fn add_new_user(&self) -> Result<(), CmdError> { - match get_user_by_name(&self.user) { - Some(_) => {} - None => { - let cmd = std::process::Command::new("useradd") - .args([ - "-m", - "-s", - &self.shell, - "-g", - &self.group, - "-c", - &self.gecos, - &self.user, - ]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `useradd -m -s {} -g {} -c {} {}`: {}", - self.shell, self.group, self.gecos, self.user, err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not add user: {} status: {}", - self.user, - cmd.status - ))); - } - } - }; - Ok(()) - } - - fn enable_passwordless_login(&self) -> Result<(), CmdError> { - let cmd = std::process::Command::new("passwd") - .args(["-d", &self.user]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `passwd -d {}`: {}", - self.user, - err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not enable password-less login: {} status: {}", - self.user, - cmd.status - ))); - } - Ok(()) - } - - fn disable_password_based_login(&self) -> Result<(), CmdError> { - let cmd = std::process::Command::new("passwd") - .args(["-N", &self.user]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `passwd -N {}`: {}", - self.user, - err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not disable password-based logins: {} status: {}", - self.user, - cmd.status - ))); - } - Ok(()) - } - - fn assign_user_profiles(&self) -> Result<(), CmdError> { - let Some(ref profiles) = self.profiles else { - return Err(CmdError::Failure(anyhow!( - "Profile list must not be empty to assign user profiles", - ))); - }; - - let mut profile_list: String = Default::default(); - for profile in profiles { - profile_list.push_str(&profile) - } - - let cmd = std::process::Command::new("usermod") - .args(["-P", &profile_list, &self.user]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `usermod -P {} {}`: {}", - profile_list, - self.user, - err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not assign user profiles: {} status: {}", - self.user, - cmd.status - ))); - } - Ok(()) - } - - fn remove_user_profiles(&self) -> Result<(), CmdError> { - let cmd = std::process::Command::new("usermod") - .args(["-P", "", &self.user]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `usermod -P '' {}`: {}", - self.user, - err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not remove user profiles: {} status: {}", - self.user, - cmd.status - ))); - } - Ok(()) - } - - fn set_up_home_directory_and_startup_files(&self) -> Result<(), CmdError> { - let Some(ref homedir) = self.homedir else { - return Err(CmdError::Failure(anyhow!( - "A home directory must be provided to set up home directory and startup files", - ))); - }; - - create_dir_all(&homedir).map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute create directory {} and its parents: {}", - &homedir, - err - )) - })?; - - let mut home_bashrc = homedir.clone(); - home_bashrc.push_str("/.bashrc"); - copy("/root/.bashrc", &home_bashrc).map_err(|err| { - CmdError::Failure(anyhow!( - "Could not copy file from /root/.bashrc to {}: {}", - &homedir, - err - )) - })?; - - let mut home_profile = homedir.clone(); - home_profile.push_str("/.profile"); - copy("/root/.profile", &home_profile).map_err(|err| { - CmdError::Failure(anyhow!( - "Could not copy file from /root/.profile to {}: {}", - &homedir, - err - )) - })?; - - // Not using std::os::unix::fs::chown here because it doesn't support - // recursive option. - let cmd = std::process::Command::new("chown") - .args(["-R", &self.user, &homedir]) - .output() - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not execute `chown -R {} {}`: {}", - self.user, - homedir, - err - )) - })?; - - if !cmd.status.success() { - return Err(CmdError::Failure(anyhow!( - "Could not change ownership: {} status: {}", - homedir, - cmd.status - ))); - } - Ok(()) - } - - fn setup_switch_zone_user(self, log: &Logger) -> Result<(), CmdError> { - info!(&log, "Add a new group for the user"; "group" => ?self.group, "user" => ?self.user); - self.add_new_group_for_user()?; - - info!(&log, "Add the user"; "user" => ?self.user, "shell" => ?self.shell, - "group" => &self.group, "gecos" => &self.gecos); - self.add_new_user()?; - - // Either enable password-less login (wicket) or disable password-based logins - // completely (support, which logs in via ssh key). - if self.nopasswd { - info!(&log, "Enable password-less login for user"; "user" => ?self.user); - self.enable_passwordless_login()?; - } else { - info!(&log, "Disable password-based logins"; "user" => ?self.user); - self.disable_password_based_login()?; - }; - - if let Some(_) = &self.profiles { - info!(&log, "Assign user profiles"; "user" => ?self.user, "profiles" => ?self.profiles); - self.assign_user_profiles()?; - } else { - info!(&log, "Remove user profiles"; "user" => ?self.user); - self.remove_user_profiles()?; - }; - - if let Some(_) = self.homedir { - info!(&log, "Set up home directory and startup files"; "user" => ?self.user, "home directory" => ?self.homedir); - self.set_up_home_directory_and_startup_files()?; - } - Ok(()) - } -} - fn parse_ip(s: &str) -> anyhow::Result { if s == "unknown" { return Err(anyhow!("ERROR: Missing input value")); @@ -571,9 +277,7 @@ async fn switch_zone_setup( "wicket".to_string(), "Wicket User".to_string(), true, - // None, "/bin/sh".to_string(), - // None, ); let support_user = SwitchZoneUser::new( @@ -581,9 +285,7 @@ async fn switch_zone_setup( "support".to_string(), "Oxide Support".to_string(), false, - // Some(String::from("/home/support")), "/bin/bash".to_string(), - // Some(vec![String::from("Primary Administrator")]), ) .with_homedir("/home/support".to_string()) .with_profiles(vec!["Primary Administrator".to_string()]); From dafe3da4791a8ad90f0342149676e217f61e0220 Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 11 Jun 2024 17:50:15 +1200 Subject: [PATCH 33/56] fmt and add files --- zone-setup/src/bin/zone-setup.rs | 5 +- zone-setup/src/lib.rs | 5 + zone-setup/src/switch_zone_user.rs | 304 +++++++++++++++++++++++++++++ 3 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 zone-setup/src/lib.rs create mode 100644 zone-setup/src/switch_zone_user.rs diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 5df204688b..5da23e98ce 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -17,10 +17,7 @@ use omicron_common::cmd::CmdError; use omicron_sled_agent::services::SWITCH_ZONE_BASEBOARD_FILE; use serde_json::Value; use slog::{info, Logger}; -use std::fs::{ - metadata, read_to_string, set_permissions, write, - OpenOptions, -}; +use std::fs::{metadata, read_to_string, set_permissions, write, OpenOptions}; use std::io::Write; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::os::unix::fs::chown; diff --git a/zone-setup/src/lib.rs b/zone-setup/src/lib.rs new file mode 100644 index 0000000000..657d414544 --- /dev/null +++ b/zone-setup/src/lib.rs @@ -0,0 +1,5 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +pub mod switch_zone_user; diff --git a/zone-setup/src/switch_zone_user.rs b/zone-setup/src/switch_zone_user.rs new file mode 100644 index 0000000000..d8dda9a599 --- /dev/null +++ b/zone-setup/src/switch_zone_user.rs @@ -0,0 +1,304 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::anyhow; +use omicron_common::cmd::CmdError; +use slog::{info, Logger}; +use std::fs::{copy, create_dir_all}; +use uzers::{get_group_by_name, get_user_by_name}; + +// User information for the switch zone +pub struct SwitchZoneUser { + user: String, + group: String, + gecos: String, + nopasswd: bool, + homedir: Option, + shell: String, + profiles: Option>, +} + +impl SwitchZoneUser { + pub fn new( + user: String, + group: String, + gecos: String, + nopasswd: bool, + shell: String, + ) -> Self { + Self { + user, + group, + gecos, + nopasswd, + homedir: None, + shell, + profiles: None, + } + } + + pub fn with_homedir(mut self, homedir: String) -> Self { + self.homedir = Some(homedir); + self + } + + pub fn with_profiles(mut self, profiles: Vec) -> Self { + self.profiles = Some(profiles); + self + } + + fn add_new_group_for_user(&self) -> Result<(), CmdError> { + match get_group_by_name(&self.group) { + Some(_) => {} + None => { + let cmd = std::process::Command::new("groupadd") + .arg(&self.group) + .output() + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute `groupadd {}`: {}", + self.group, + err + )) + })?; + + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not add group: {} status: {}", + self.group, + cmd.status + ))); + } + } + }; + Ok(()) + } + + fn add_new_user(&self) -> Result<(), CmdError> { + match get_user_by_name(&self.user) { + Some(_) => {} + None => { + let cmd = std::process::Command::new("useradd") + .args([ + "-m", + "-s", + &self.shell, + "-g", + &self.group, + "-c", + &self.gecos, + &self.user, + ]) + .output() + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute `useradd -m -s {} -g {} -c {} {}`: {}", + self.shell, self.group, self.gecos, self.user, err + )) + })?; + + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not add user: {} status: {}", + self.user, + cmd.status + ))); + } + } + }; + Ok(()) + } + + fn enable_passwordless_login(&self) -> Result<(), CmdError> { + let cmd = std::process::Command::new("passwd") + .args(["-d", &self.user]) + .output() + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute `passwd -d {}`: {}", + self.user, + err + )) + })?; + + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not enable password-less login: {} status: {}", + self.user, + cmd.status + ))); + } + Ok(()) + } + + fn disable_password_based_login(&self) -> Result<(), CmdError> { + let cmd = std::process::Command::new("passwd") + .args(["-N", &self.user]) + .output() + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute `passwd -N {}`: {}", + self.user, + err + )) + })?; + + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not disable password-based logins: {} status: {}", + self.user, + cmd.status + ))); + } + Ok(()) + } + + fn assign_user_profiles(&self) -> Result<(), CmdError> { + let Some(ref profiles) = self.profiles else { + return Err(CmdError::Failure(anyhow!( + "Profile list must not be empty to assign user profiles", + ))); + }; + + let mut profile_list: String = Default::default(); + for profile in profiles { + profile_list.push_str(&profile) + } + + let cmd = std::process::Command::new("usermod") + .args(["-P", &profile_list, &self.user]) + .output() + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute `usermod -P {} {}`: {}", + profile_list, + self.user, + err + )) + })?; + + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not assign user profiles: {} status: {}", + self.user, + cmd.status + ))); + } + Ok(()) + } + + fn remove_user_profiles(&self) -> Result<(), CmdError> { + let cmd = std::process::Command::new("usermod") + .args(["-P", "", &self.user]) + .output() + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute `usermod -P '' {}`: {}", + self.user, + err + )) + })?; + + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not remove user profiles: {} status: {}", + self.user, + cmd.status + ))); + } + Ok(()) + } + + fn set_up_home_directory_and_startup_files(&self) -> Result<(), CmdError> { + let Some(ref homedir) = self.homedir else { + return Err(CmdError::Failure(anyhow!( + "A home directory must be provided to set up home directory and startup files", + ))); + }; + + create_dir_all(&homedir).map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute create directory {} and its parents: {}", + &homedir, + err + )) + })?; + + let mut home_bashrc = homedir.clone(); + home_bashrc.push_str("/.bashrc"); + copy("/root/.bashrc", &home_bashrc).map_err(|err| { + CmdError::Failure(anyhow!( + "Could not copy file from /root/.bashrc to {}: {}", + &homedir, + err + )) + })?; + + let mut home_profile = homedir.clone(); + home_profile.push_str("/.profile"); + copy("/root/.profile", &home_profile).map_err(|err| { + CmdError::Failure(anyhow!( + "Could not copy file from /root/.profile to {}: {}", + &homedir, + err + )) + })?; + + // Not using std::os::unix::fs::chown here because it doesn't support + // recursive option. + let cmd = std::process::Command::new("chown") + .args(["-R", &self.user, &homedir]) + .output() + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not execute `chown -R {} {}`: {}", + self.user, + homedir, + err + )) + })?; + + if !cmd.status.success() { + return Err(CmdError::Failure(anyhow!( + "Could not change ownership: {} status: {}", + homedir, + cmd.status + ))); + } + Ok(()) + } + + pub fn setup_switch_zone_user(self, log: &Logger) -> Result<(), CmdError> { + info!(&log, "Add a new group for the user"; "group" => ?self.group, "user" => ?self.user); + self.add_new_group_for_user()?; + + info!(&log, "Add the user"; "user" => ?self.user, "shell" => ?self.shell, + "group" => &self.group, "gecos" => &self.gecos); + self.add_new_user()?; + + // Either enable password-less login (wicket) or disable password-based logins + // completely (support, which logs in via ssh key). + if self.nopasswd { + info!(&log, "Enable password-less login for user"; "user" => ?self.user); + self.enable_passwordless_login()?; + } else { + info!(&log, "Disable password-based logins"; "user" => ?self.user); + self.disable_password_based_login()?; + }; + + if let Some(_) = &self.profiles { + info!(&log, "Assign user profiles"; "user" => ?self.user, "profiles" => ?self.profiles); + self.assign_user_profiles()?; + } else { + info!(&log, "Remove user profiles"; "user" => ?self.user); + self.remove_user_profiles()?; + }; + + if let Some(_) = self.homedir { + info!(&log, "Set up home directory and startup files"; "user" => ?self.user, "home directory" => ?self.homedir); + self.set_up_home_directory_and_startup_files()?; + } + Ok(()) + } +} From 2ea71b8fd613c676e9a17e4ac0453a5750da1092 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 12 Jun 2024 11:46:58 +1200 Subject: [PATCH 34/56] Add logs from PR #5853 --- sled-agent/src/services.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 621011f85f..b79e7a9e79 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -4447,6 +4447,7 @@ impl ServiceManager { } }; + info!(self.inner.log, "Setting up uplinkd service"); let smfh = SmfHelper::new(&zone, &SwitchService::Uplink); // We want to delete all the properties in the `uplinks` group, but we @@ -4457,6 +4458,7 @@ impl ServiceManager { for port_config in &our_ports { for addr in &port_config.addrs { + info!(self.inner.log, "configuring port: {port_config:?}"); smfh.addpropvalue_type( &format!("uplinks/{}_0", port_config.port,), &addr.to_string(), From 06e161409890fb6e428a8de255dbefb7b9b92852 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 12 Jun 2024 12:59:26 +1200 Subject: [PATCH 35/56] Tidy up --- illumos-utils/src/route.rs | 21 --------------------- sled-agent/src/services.rs | 26 ++++++++++---------------- zone-setup/src/bin/zone-setup.rs | 12 ++++++++++++ 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/illumos-utils/src/route.rs b/illumos-utils/src/route.rs index df9d0c002f..ceff2b3d9e 100644 --- a/illumos-utils/src/route.rs +++ b/illumos-utils/src/route.rs @@ -107,25 +107,4 @@ impl Route { }; Ok(()) } - - // TODO: Perhaps get rid of this function? Or should I continue using the - // running zone method? - pub fn add_bootstrap_route( - bootstrap_prefix: u16, - gz_bootstrap_addr: Ipv6Addr, - zone_vnic_name: &str, - ) -> Result<(), ExecutionError> { - let mut cmd = std::process::Command::new(PFEXEC); - let cmd = cmd.args(&[ - ROUTE, - "add", - "-inet6", - &format!("{bootstrap_prefix:x}::/16"), - &gz_bootstrap_addr.to_string(), - "-ifp", - zone_vnic_name, - ]); - execute(cmd)?; - Ok(()) - } } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index b79e7a9e79..7ba9ec0f94 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1550,7 +1550,7 @@ impl ServiceManager { if let Some(uuid) = unique_name { zone_builder = zone_builder.with_unique_name(uuid); } - if let Some(vnic) = bootstrap_vnic.clone() { + if let Some(vnic) = bootstrap_vnic { zone_builder = zone_builder.with_bootstrap_vnic(vnic); } let installed_zone = zone_builder @@ -1598,13 +1598,13 @@ impl ServiceManager { return Err(Error::SledAgentNotReady); }; - let listen_addr = underlay_address; + let listen_addr = *underlay_address; let listen_port = &CLICKHOUSE_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![*underlay_address], + &vec![listen_addr], )?; let dns_service = Self::dns_install(info, None, &None).await?; @@ -1651,13 +1651,13 @@ impl ServiceManager { return Err(Error::SledAgentNotReady); }; - let listen_addr = underlay_address; + let listen_addr = *underlay_address; let listen_port = &CLICKHOUSE_KEEPER_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![*underlay_address], + &vec![listen_addr], )?; let dns_service = Self::dns_install(info, None, &None).await?; @@ -1778,13 +1778,13 @@ impl ServiceManager { let Some(info) = self.inner.sled_info.get() else { return Err(Error::SledAgentNotReady); }; - let listen_addr = &underlay_address; + let listen_addr = *underlay_address; let listen_port = &CRUCIBLE_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![*underlay_address], + &vec![listen_addr], )?; let dataset_name = DatasetName::new( @@ -1837,13 +1837,13 @@ impl ServiceManager { return Err(Error::SledAgentNotReady); }; - let listen_addr = &underlay_address; + let listen_addr = *underlay_address; let listen_port = &CRUCIBLE_PANTRY_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![*underlay_address], + &vec![listen_addr], )?; let config = PropertyGroupBuilder::new("config") @@ -2967,7 +2967,6 @@ impl ServiceManager { .add_service(pumpkind_service) .add_service(mgd_service) .add_service(mg_ddm_service) - // TODO: I am unsure if I should add this service to the property builder .add_service(sp_sim_service) .add_service(uplink_service); profile @@ -2976,11 +2975,9 @@ impl ServiceManager { .map_err(|err| { Error::io("Failed to setup Switch zone profile", err) })?; - // return Ok(RunningZone::boot(installed_zone).await?); } } - // // TODO: Remove from here until end let running_zone = RunningZone::boot(installed_zone).await?; // Part of the process to ensure bootstrap address is to set up @@ -3015,6 +3012,7 @@ impl ServiceManager { })?; } Ok(running_zone) + // TODO: Remove from here until end // // let addresses = match &request { // ZoneArgs::Omicron(OmicronZoneConfigLocal { @@ -4595,10 +4593,6 @@ impl ServiceManager { match service { SwitchService::ManagementGatewayService => { info!(self.inner.log, "configuring MGS service"); - // TODO: Make sure all services use delpropvalue_default_instance - // and addpropvalue_default_instance instead of delpropvalue - // and addpropvalue. Verify property values are correct - // Remove any existing `config/address` values // without deleting the property itself. smfh.delpropvalue_default_instance( diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 5da23e98ce..f55cbc1339 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -602,11 +602,23 @@ async fn common_nw_set_up( .map_err(|err| CmdError::Failure(anyhow!(err)))?; // TODO: Log if there are no addresses, or add a flag so that this doesn't run on the switch zone + if static_addrs.is_empty() { + info!( + &log, + "No static addresses provided, will not ensure static and auto-configured addresses are set on the IP interface" + ); + } + for addr in &static_addrs { if **addr != Ipv6Addr::LOCALHOST { info!(&log, "Ensuring static and auto-configured addresses are set on the IP interface"; "data link" => ?datalink, "static address" => ?addr); Ipadm::create_static_and_autoconfigured_addrs(&datalink, addr) .map_err(|err| CmdError::Failure(anyhow!(err)))?; + } else { + info!( + &log, + "Static address is localhost, will not ensure it's set on the IP interface" + ); } } From 6221c39b71adefdfc6f2926497045e11d84d4181 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 12 Jun 2024 13:35:22 +1200 Subject: [PATCH 36/56] remove switch zone setup bash script entirely --- package-manifest.toml | 1 - smf/switch_zone_setup/switch_zone_setup | 55 ------------------------- 2 files changed, 56 deletions(-) delete mode 100755 smf/switch_zone_setup/switch_zone_setup diff --git a/package-manifest.toml b/package-manifest.toml index 7ffb9c563c..78ef430492 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -439,7 +439,6 @@ service_name = "switch_zone_setup" source.type = "local" source.paths = [ { from = "smf/switch_zone_setup/manifest.xml", to = "/var/svc/manifest/site/switch_zone_setup/manifest.xml" }, - { from = "smf/switch_zone_setup/switch_zone_setup", to = "/opt/oxide/bin/switch_zone_setup" }, { from = "smf/switch_zone_setup/support_authorized_keys", to = "/opt/oxide/support/authorized_keys" }, { from = "/opt/ooce/pgsql-13/lib/amd64", to = "/opt/ooce/pgsql-13/lib/amd64" }, ] diff --git a/smf/switch_zone_setup/switch_zone_setup b/smf/switch_zone_setup/switch_zone_setup deleted file mode 100755 index ee9a28b1b8..0000000000 --- a/smf/switch_zone_setup/switch_zone_setup +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/ksh - -set -ex -o pipefail - -. /lib/svc/share/smf_include.sh - -# TODO: Remove this file if we are confident we'll keep using the set up CLI -BASEBOARD_INFO="$(svcprop -c -p config/baseboard-info "${SMF_FMRI}")" - -# set up the users required for wicket and support. -USERS=( - (user=wicket group=wicket gecos='Wicket User' nopasswd=1 shell='/bin/sh') - (user=support group=support gecos='Oxide Support' homedir='/home/support' - shell='/bin/bash' profiles=('Primary Administrator') - ) -) - -for i in "${!USERS[@]}"; do - nameref u=USERS[$i] - - # Add a new group for the user. - getent group "${u.group}" >/dev/null 2>&1 || groupadd "${u.group}" - # Add the user. - getent passwd "${u.user}" >/dev/null 2>&1 || \ - useradd -m -s "${u.shell}" -g "${u.group}" -c "${u.gecos}" \ - "${u.user}" - - # Either enable passwordless login (wicket) or disable password-based logins - # completely (support, which logs in via ssh key). - if ((u.nopasswd)); then - passwd -d "${u.user}" - else - passwd -N "${u.user}" - fi - - # Assign or remove profiles - if [[ -n "${u.profiles}" ]]; then - usermod -P"$(printf '%s,' "${u.profiles[@]}")" "${u.user}" - else - usermod -P '' "${u.user}" - fi - - if [[ -n "${u.homedir}" ]]; then - mkdir -p "${u.homedir}" - for f in .bashrc .profile; do - cp "/root/$f" "${u.homedir}/$f" - done - chown -R "${u.user}" "${u.homedir}" - fi -done - -# TODO: Eventually we'll want all of the above code to be part of the zone-setup CLI as well -exec /opt/oxide/zone-setup-cli/bin/zone-setup switch-zone -i "$BASEBOARD_INFO" - -exit $SMF_EXIT_OK From 19bd61ccee3de5c2566893dd0ed2b78a565888f9 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 12 Jun 2024 16:52:06 +1200 Subject: [PATCH 37/56] Verify ensure default route loop --- illumos-utils/src/lib.rs | 2 +- sled-agent/src/services.rs | 2 -- zone-setup/src/bin/zone-setup.rs | 49 +++++++++++++++++++------------- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/illumos-utils/src/lib.rs b/illumos-utils/src/lib.rs index d041c866b0..c5db9ed7de 100644 --- a/illumos-utils/src/lib.rs +++ b/illumos-utils/src/lib.rs @@ -36,7 +36,7 @@ pub const PFEXEC: &str = "/usr/bin/pfexec"; pub struct CommandFailureInfo { command: String, status: std::process::ExitStatus, - stdout: String, + pub stdout: String, stderr: String, } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 7ba9ec0f94..47dbd47bff 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -2364,8 +2364,6 @@ impl ServiceManager { let mut switch_zone_setup_config = PropertyGroupBuilder::new("config"); - // TODO: investigate links_need_link_local more throroughly, there don't seem to be any - // in either of the scrimlets for (link, needs_link_local) in installed_zone.links().iter().zip(links_need_link_local) { diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index f55cbc1339..3b11295dda 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -11,6 +11,7 @@ use illumos_utils::ipadm::Ipadm; use illumos_utils::route::{Gateway, Route}; use illumos_utils::svcadm::Svcadm; use illumos_utils::zone::Zones; +use illumos_utils::ExecutionError; use omicron_common::backoff::{retry_notify, retry_policy_local, BackoffError}; use omicron_common::cmd::fatal; use omicron_common::cmd::CmdError; @@ -601,7 +602,6 @@ async fn common_nw_set_up( Ipadm::set_interface_mtu(&datalink) .map_err(|err| CmdError::Failure(anyhow!(err)))?; - // TODO: Log if there are no addresses, or add a flag so that this doesn't run on the switch zone if static_addrs.is_empty() { info!( &log, @@ -611,39 +611,50 @@ async fn common_nw_set_up( for addr in &static_addrs { if **addr != Ipv6Addr::LOCALHOST { - info!(&log, "Ensuring static and auto-configured addresses are set on the IP interface"; "data link" => ?datalink, "static address" => ?addr); + info!( + &log, + "Ensuring static and auto-configured addresses are set on the IP interface"; + "data link" => ?datalink, + "static address" => ?addr, + ); Ipadm::create_static_and_autoconfigured_addrs(&datalink, addr) .map_err(|err| CmdError::Failure(anyhow!(err)))?; } else { info!( &log, "Static address is localhost, will not ensure it's set on the IP interface" - ); + ); } } - // TODO: Run this enough times to make sure this implementation is solid - // perhaps somehow find out a way to know when the gateway is up? - // perhaps configure this for the switch zone separately? - // - // Maybe instead of waiting we want to implement the "Maybe gateway" logic, - // and afterwards update this when the other services refresh? - // - // NOTE: Route must come after bootstrap, meaning this won't succeed until - // after all bootstrap address code is run in switch zone. Maybe this service - // can depend on the switch setup serice? + // NOTE: Ensuring default route with gateway must happen after peer agents have been initialized. + // Omicron zones will be able ensure a default route with gateway immediately, but the switch zone + // on the secondary scrimlet might need a few tries while it waits. retry_notify( - // TODO: Is this the best retry policy? retry_policy_local(), || async { info!(&log, "Ensuring there is a default route"; "gateway" => ?gateway); Route::ensure_default_route_with_gateway(Gateway::Ipv6(gateway)) .map_err(|err| { - // TODO: Only make transient for the following error: - // executed and failed with status: exit status: 128 stdout: add net default: gateway fd00:1122:3344:101::1: Network is unreachable\n stderr: - BackoffError::transient( - CmdError::Failure(anyhow!(err)), - )}) + match err { + ExecutionError::CommandFailure(ref e) => { + if e.stdout.contains("Network is unreachable") { + BackoffError::transient( + CmdError::Failure(anyhow!(err)), + ) + } else { + BackoffError::permanent( + CmdError::Failure(anyhow!(err)), + ) + } + } + _ => { + BackoffError::permanent( + CmdError::Failure(anyhow!(err)), + ) + } + } + }) }, |err, delay| { info!( From 1bd8e85daf9e5c33ec74b0c6e4935b0c927fe7db Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 12 Jun 2024 17:03:18 +1200 Subject: [PATCH 38/56] Remove more commented out code --- sled-agent/src/services.rs | 591 ------------------------------------- 1 file changed, 591 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 47dbd47bff..72c0855abb 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -3010,585 +3010,6 @@ impl ServiceManager { })?; } Ok(running_zone) - // TODO: Remove from here until end - // - // let addresses = match &request { - // ZoneArgs::Omicron(OmicronZoneConfigLocal { - // zone: OmicronZoneConfig { underlay_address, .. }, - // .. - // }) => std::slice::from_ref(underlay_address), - // ZoneArgs::Switch(req) => &req.zone.addresses, - // }; - // for addr in addresses { - // if *addr == Ipv6Addr::LOCALHOST { - // continue; - // } - // info!( - // self.inner.log, - // "Ensuring address {} exists", - // addr.to_string() - // ); - // let addr_request = - // AddressRequest::new_static(IpAddr::V6(*addr), None); - // running_zone.ensure_address(addr_request).await?; - // info!( - // self.inner.log, - // "Ensuring address {} exists - OK", - // addr.to_string() - // ); - // } - - // let maybe_gateway = if let Some(info) = self.inner.sled_info.get() { - // // Only consider a route to the sled's underlay address if the - // // underlay is up. - // let sled_underlay_subnet = - // Ipv6Subnet::::new(info.underlay_address); - // - // if addresses - // .iter() - // .any(|ip| sled_underlay_subnet.net().contains(*ip)) - // { - // // If the underlay is up, provide a route to it through an - // // existing address in the Zone on the same subnet. - // info!(self.inner.log, "Zone using sled underlay as gateway"); - // Some(info.underlay_address) - // } else { - // // If no such address exists in the sled's subnet, don't route - // // to anything. - // info!( - // self.inner.log, - // "Zone not using gateway (even though underlay is up)" - // ); - // None - // } - // } else { - // // If the underlay doesn't exist, no routing occurs. - // info!( - // self.inner.log, - // "Zone not using gateway (underlay is not up)" - // ); - // None - // }; - - // let sidecar_revision = match &self.inner.sidecar_revision { - // SidecarRevision::Physical(rev) => rev.to_string(), - // SidecarRevision::SoftZone(rev) - // | SidecarRevision::SoftPropolis(rev) => format!( - // "softnpu_front_{}_rear_{}", - // rev.front_port_count, rev.rear_port_count - // ), - // }; - - // if let Some(gateway) = maybe_gateway { - // running_zone.add_default_route(gateway).map_err(|err| { - // Error::ZoneCommand { intent: "Adding Route".to_string(), err } - // })?; - // } - - // match &request { - // ZoneArgs::Omicron(zone_config) => { - // match &zone_config.zone.zone_type { - // OmicronZoneType::BoundaryNtp { .. } - // | OmicronZoneType::Clickhouse { .. } - // | OmicronZoneType::ClickhouseKeeper { .. } - // | OmicronZoneType::CockroachDb { .. } - // | OmicronZoneType::Crucible { .. } - // | OmicronZoneType::CruciblePantry { .. } - // | OmicronZoneType::ExternalDns { .. } - // | OmicronZoneType::InternalDns { .. } - // | OmicronZoneType::InternalNtp { .. } - // | OmicronZoneType::Nexus { .. } - // | OmicronZoneType::Oximeter { .. } => { - // panic!( - // "{} is a service which exists as part of a \ - // self-assembling zone", - // &zone_config.zone.zone_type.zone_type_str(), - // ) - // } - // }; - // } - // ZoneArgs::Switch(request) => { - // for service in &request.zone.services { - // // TODO: Related to - // // https://github.com/oxidecomputer/omicron/pull/1124 , should we - // // avoid importing this manifest? - // debug!(self.inner.log, "importing manifest"); - // - // let smfh = SmfHelper::new(&running_zone, service); - // smfh.import_manifest()?; - // - // match service { - // SwitchService::ManagementGatewayService => { - // info!(self.inner.log, "Setting up MGS service"); - // smfh.setprop("config/id", request.zone.id)?; - // - // // Always tell MGS to listen on localhost so wicketd - // // can contact it even before we have an underlay - // // network. - // smfh.addpropvalue( - // "config/address", - // &format!("[::1]:{MGS_PORT}"), - // )?; - // - // if let Some(address) = request.zone.addresses.get(0) - // { - // // Don't use localhost twice - // if *address != Ipv6Addr::LOCALHOST { - // smfh.addpropvalue( - // "config/address", - // &format!("[{address}]:{MGS_PORT}"), - // )?; - // } - // } - // - // if let Some(info) = self.inner.sled_info.get() { - // smfh.setprop("config/rack_id", info.rack_id)?; - // } - // - // smfh.refresh()?; - // } - // SwitchService::SpSim => { - // info!( - // self.inner.log, - // "Setting up Simulated SP service" - // ); - // } - // SwitchService::Wicketd { baseboard } => { - // info!(self.inner.log, "Setting up wicketd service"); - // - // smfh.setprop( - // "config/address", - // &format!("[::1]:{WICKETD_PORT}"), - // )?; - // - // // If we're launching the switch zone, we'll have a - // // bootstrap_address based on our call to - // // `self.bootstrap_address_needed` (which always - // // gives us an address for the switch zone. If we - // // _don't_ have a bootstrap address, someone has - // // requested wicketd in a non-switch zone; return an - // // error. - // let Some((_, bootstrap_address)) = - // bootstrap_name_and_address - // else { - // return Err(Error::BadServiceRequest { - // service: "wicketd".to_string(), - // message: concat!( - // "missing bootstrap address: ", - // "wicketd can only be started in the ", - // "switch zone", - // ) - // .to_string(), - // }); - // }; - // smfh.setprop( - // "config/artifact-address", - // &format!( - // "[{bootstrap_address}]:{BOOTSTRAP_ARTIFACT_PORT}" - // ), - // )?; - // - // smfh.setprop( - // "config/mgs-address", - // &format!("[::1]:{MGS_PORT}"), - // )?; - // - // // We intentionally bind `nexus-proxy-address` to - // // `::` so wicketd will serve this on all - // // interfaces, particularly the tech port - // // interfaces, allowing external clients to connect - // // to this Nexus proxy. - // smfh.setprop( - // "config/nexus-proxy-address", - // &format!("[::]:{WICKETD_NEXUS_PROXY_PORT}"), - // )?; - // if let Some(underlay_address) = self - // .inner - // .sled_info - // .get() - // .map(|info| info.underlay_address) - // { - // let rack_subnet = Ipv6Subnet::::new( - // underlay_address, - // ); - // smfh.setprop( - // "config/rack-subnet", - // &rack_subnet.net().ip().to_string(), - // )?; - // } - // - // let serialized_baseboard = - // serde_json::to_string_pretty(&baseboard)?; - // let serialized_baseboard_path = - // Utf8PathBuf::from(format!( - // "{}/opt/oxide/baseboard.json", - // running_zone.root() - // )); - // tokio::fs::write( - // &serialized_baseboard_path, - // &serialized_baseboard, - // ) - // .await - // .map_err(|err| { - // Error::io_path(&serialized_baseboard_path, err) - // })?; - // smfh.setprop( - // "config/baseboard-file", - // String::from("/opt/oxide/baseboard.json"), - // )?; - // - // smfh.refresh()?; - // } - // SwitchService::Dendrite { asic } => { - // info!( - // self.inner.log, - // "Setting up dendrite service" - // ); - // - // if let Some(info) = self.inner.sled_info.get() { - // smfh.setprop("config/rack_id", info.rack_id)?; - // smfh.setprop( - // "config/sled_id", - // info.config.sled_id, - // )?; - // } else { - // info!( - // self.inner.log, - // "no rack_id/sled_id available yet" - // ); - // } - // - // smfh.delpropvalue("config/address", "*")?; - // smfh.delpropvalue("config/dns_server", "*")?; - // for address in &request.zone.addresses { - // smfh.addpropvalue( - // "config/address", - // &format!("[{}]:{}", address, DENDRITE_PORT), - // )?; - // if *address != Ipv6Addr::LOCALHOST { - // let az_prefix = - // Ipv6Subnet::::new(*address); - // for addr in - // Resolver::servers_from_subnet(az_prefix) - // { - // smfh.addpropvalue( - // "config/dns_server", - // &format!("{addr}"), - // )?; - // } - // } - // } - // match asic { - // DendriteAsic::TofinoAsic => { - // // There should be exactly one device_name - // // associated with this zone: the /dev path - // // for the tofino ASIC. - // let dev_cnt = device_names.len(); - // if dev_cnt == 1 { - // smfh.setprop( - // "config/dev_path", - // device_names[0].clone(), - // )?; - // } else { - // return Err(Error::SledLocalZone( - // anyhow::anyhow!( - // "{dev_cnt} devices needed \ - // for tofino asic" - // ), - // )); - // } - // smfh.setprop( - // "config/port_config", - // "/opt/oxide/dendrite/misc/sidecar_config.toml", - // )?; - // smfh.setprop("config/board_rev", &sidecar_revision)?; - // } - // DendriteAsic::TofinoStub => smfh.setprop( - // "config/port_config", - // "/opt/oxide/dendrite/misc/model_config.toml", - // )?, - // asic @ (DendriteAsic::SoftNpuZone - // | DendriteAsic::SoftNpuPropolisDevice) => { - // if asic == &DendriteAsic::SoftNpuZone { - // smfh.setprop("config/mgmt", "uds")?; - // smfh.setprop( - // "config/uds_path", - // "/opt/softnpu/stuff", - // )?; - // } - // if asic == &DendriteAsic::SoftNpuPropolisDevice { - // smfh.setprop("config/mgmt", "uart")?; - // } - // let s = match self.inner.sidecar_revision { - // SidecarRevision::SoftZone(ref s) => s, - // SidecarRevision::SoftPropolis(ref s) => s, - // _ => { - // return Err(Error::SidecarRevision( - // anyhow::anyhow!( - // "expected soft sidecar \ - // revision" - // ), - // )) - // } - // }; - // smfh.setprop( - // "config/front_ports", - // &s.front_port_count.to_string(), - // )?; - // smfh.setprop( - // "config/rear_ports", - // &s.rear_port_count.to_string(), - // )?; - // smfh.setprop( - // "config/port_config", - // "/opt/oxide/dendrite/misc/softnpu_single_sled_config.toml", - // )? - // } - // }; - // smfh.refresh()?; - // } - // SwitchService::Tfport { pkt_source, asic } => { - // info!(self.inner.log, "Setting up tfport service"); - // - // let is_gimlet = is_gimlet().map_err(|e| { - // Error::Underlay( - // underlay::Error::SystemDetection(e), - // ) - // })?; - // - // if is_gimlet { - // // Collect the prefixes for each techport. - // let nameaddr = - // bootstrap_name_and_address.as_ref(); - // let techport_prefixes = match nameaddr { - // Some((_, addr)) => { - // Self::bootstrap_addr_to_techport_prefixes(addr) - // } - // None => { - // return Err(Error::BadServiceRequest { - // service: "tfport".into(), - // message: "bootstrap addr missing" - // .into(), - // }); - // } - // }; - // - // for (i, prefix) in - // techport_prefixes.into_iter().enumerate() - // { - // // Each `prefix` is an `Ipv6Subnet` - // // including a netmask. Stringify just the - // // network address, without the mask. - // smfh.setprop( - // format!("config/techport{i}_prefix"), - // prefix.net().network().to_string(), - // )?; - // } - // smfh.setprop("config/pkt_source", pkt_source)?; - // } - // if asic == &DendriteAsic::SoftNpuZone { - // smfh.setprop("config/flags", "--sync-only")?; - // } - // if asic == &DendriteAsic::SoftNpuPropolisDevice { - // smfh.setprop("config/pkt_source", pkt_source)?; - // } - // smfh.setprop( - // "config/host", - // &format!("[{}]", Ipv6Addr::LOCALHOST), - // )?; - // smfh.setprop( - // "config/port", - // &format!("{}", DENDRITE_PORT), - // )?; - // - // smfh.refresh()?; - // } - // SwitchService::Lldpd { baseboard } => { - // info!(self.inner.log, "Setting up lldpd service"); - // - // match baseboard { - // Baseboard::Gimlet { - // identifier, model, .. - // } - // | Baseboard::Pc { identifier, model, .. } => { - // smfh.setprop( - // "config/scrimlet_id", - // identifier, - // )?; - // smfh.setprop( - // "config/scrimlet_model", - // model, - // )?; - // } - // Baseboard::Unknown => {} - // } - // smfh.setprop( - // "config/board_rev", - // &sidecar_revision, - // )?; - // - // smfh.delpropvalue("config/address", "*")?; - // for address in &request.zone.addresses { - // smfh.addpropvalue( - // "config/address", - // &format!("[{}]:{}", address, LLDP_PORT), - // )?; - // } - // smfh.refresh()?; - // } - // SwitchService::Pumpkind { asic } => { - // // The pumpkin daemon is only needed when running on - // // with real sidecar. - // if asic == &DendriteAsic::TofinoAsic { - // info!( - // self.inner.log, - // "Setting up pumpkind service" - // ); - // smfh.setprop("config/mode", "switch")?; - // smfh.refresh()?; - // } - // } - // SwitchService::Uplink => { - // // Nothing to do here - this service is special and - // // configured in - // // `ensure_switch_zone_uplinks_configured` - // } - // SwitchService::Mgd => { - // info!(self.inner.log, "Setting up mgd service"); - // smfh.delpropvalue("config/dns_servers", "*")?; - // if let Some(info) = self.inner.sled_info.get() { - // smfh.setprop("config/rack_uuid", info.rack_id)?; - // smfh.setprop( - // "config/sled_uuid", - // info.config.sled_id, - // )?; - // } - // for address in &request.zone.addresses { - // if *address != Ipv6Addr::LOCALHOST { - // let az_prefix = - // Ipv6Subnet::::new(*address); - // for addr in - // Resolver::servers_from_subnet(az_prefix) - // { - // smfh.addpropvalue( - // "config/dns_servers", - // &format!("{addr}"), - // )?; - // } - // break; - // } - // } - // smfh.refresh()?; - // } - // SwitchService::MgDdm { mode } => { - // info!(self.inner.log, "Setting up mg-ddm service"); - // smfh.setprop("config/mode", &mode)?; - // if let Some(info) = self.inner.sled_info.get() { - // smfh.setprop("config/rack_uuid", info.rack_id)?; - // smfh.setprop( - // "config/sled_uuid", - // info.config.sled_id, - // )?; - // } - // smfh.delpropvalue("config/dns_servers", "*")?; - // for address in &request.zone.addresses { - // if *address != Ipv6Addr::LOCALHOST { - // let az_prefix = - // Ipv6Subnet::::new(*address); - // for addr in - // Resolver::servers_from_subnet(az_prefix) - // { - // smfh.addpropvalue( - // "config/dns_servers", - // &format!("{addr}"), - // )?; - // } - // break; - // } - // } - // - // let is_gimlet = is_gimlet().map_err(|e| { - // Error::Underlay( - // underlay::Error::SystemDetection(e), - // ) - // })?; - // - // let maghemite_interfaces: Vec = - // if is_gimlet { - // (0..32) - // .map(|i| { - // // See the `tfport_name` function - // // for how tfportd names the - // // addrconf it creates. Right now, - // // that's `tfportrear[0-31]_0` for - // // all rear ports, which is what - // // we're directing ddmd to listen - // // for advertisements on. - // // - // // This may grow in a multi-rack - // // future to include a subset of - // // "front" ports too, when racks are - // // cabled together. - // AddrObject::new( - // &format!("tfportrear{}_0", i), - // IPV6_LINK_LOCAL_NAME, - // ) - // .unwrap() - // }) - // .collect() - // } else { - // self.inner - // .switch_zone_maghemite_links - // .iter() - // .map(|i| { - // AddrObject::new( - // &i.to_string(), - // IPV6_LINK_LOCAL_NAME, - // ) - // .unwrap() - // }) - // .collect() - // }; - // - // smfh.setprop( - // "config/interfaces", - // // `svccfg setprop` requires a list of values to - // // be enclosed in `()`, and each string value to - // // be enclosed in `""`. Note that we do _not_ - // // need to escape the parentheses, since this is - // // not passed through a shell, but directly to - // // `exec(2)` in the zone. - // format!( - // "({})", - // maghemite_interfaces - // .iter() - // .map(|interface| format!( - // r#""{}""#, - // interface - // )) - // .join(" "), - // ), - // )?; - // - // if is_gimlet { - // // Ddm for a scrimlet needs to be configured to - // // talk to dendrite - // smfh.setprop("config/dpd_host", "[::1]")?; - // smfh.setprop("config/dpd_port", DENDRITE_PORT)?; - // } - // smfh.setprop("config/dendrite", "true")?; - // - // smfh.refresh()?; - // } - // } - // - // debug!(self.inner.log, "enabling service"); - // smfh.enable()?; - // } - // } - // }; - // - // Ok(running_zone) } // Ensures that a single Omicron zone is running. @@ -4554,7 +3975,6 @@ impl ServiceManager { .map(|addr| addr.to_string()) .unwrap_or_else(|| "".to_string()); - // TODO: I should probably add this to the zone_network_setup_service as part of refresh for addr in &request.addresses { if *addr == Ipv6Addr::LOCALHOST { continue; @@ -4574,17 +3994,6 @@ impl ServiceManager { ); } - // TODO: This is probably not necessary as it's added by the zone_network_setup service - // Or perhaps I should add this to a refresh method on the zone_network_setup service? - // if let Some(info) = self.inner.sled_info.get() { - // zone.add_default_route(info.underlay_address).map_err( - // |err| Error::ZoneCommand { - // intent: "Adding Route".to_string(), - // err, - // }, - // )?; - // } - for service in &request.services { let smfh = SmfHelper::new(&zone, service); From d2f7ae0c81d4524921437edfe8bd57f7e5fc0e00 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 12 Jun 2024 17:32:35 +1200 Subject: [PATCH 39/56] Remove wicket service's dependency on common networking service --- smf/wicketd/manifest.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/smf/wicketd/manifest.xml b/smf/wicketd/manifest.xml index 6d3e8a238f..349d7b5a57 100644 --- a/smf/wicketd/manifest.xml +++ b/smf/wicketd/manifest.xml @@ -11,12 +11,6 @@ - - - - - From 30eb1ee38987201ac71f2115fdd89da4b08710c7 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 13 Jun 2024 13:30:26 +1200 Subject: [PATCH 40/56] Make sure to set all properties on instance FMRI --- sled-agent/src/services.rs | 75 ++++++++++++++++++++++++++---------- sled-agent/src/smf_helper.rs | 70 +++++++++++---------------------- 2 files changed, 78 insertions(+), 67 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 72c0855abb..dd30b70814 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -4009,22 +4009,27 @@ impl ServiceManager { // Restore the localhost address that we always add // when setting up MGS. - smfh.addpropvalue_default_instance( + smfh.addpropvalue_type_default_instance( "config/address", &format!("[::1]:{MGS_PORT}"), + "astring", )?; // Add the underlay address. - smfh.addpropvalue_default_instance( + smfh.addpropvalue_type_default_instance( "config/address", &format!("[{address}]:{MGS_PORT}"), + "astring", )?; // It should be impossible for the `sled_info` not // to be set here, as the underlay is set at the // same time. if let Some(info) = self.inner.sled_info.get() { - smfh.setprop("config/rack_id", info.rack_id)?; + smfh.setprop_default_instance( + "config/rack_id", + info.rack_id, + )?; } else { error!( self.inner.log, @@ -4047,8 +4052,11 @@ impl ServiceManager { "configuring dendrite service" ); if let Some(info) = self.inner.sled_info.get() { - smfh.setprop("config/rack_id", info.rack_id)?; - smfh.setprop( + smfh.setprop_default_instance( + "config/rack_id", + info.rack_id, + )?; + smfh.setprop_default_instance( "config/sled_id", info.config.sled_id, )?; @@ -4062,11 +4070,15 @@ impl ServiceManager { "config/address", "*", )?; - smfh.delpropvalue("config/dns_server", "*")?; + smfh.delpropvalue_default_instance( + "config/dns_server", + "*", + )?; for address in &request.addresses { - smfh.addpropvalue_default_instance( + smfh.addpropvalue_type_default_instance( "config/address", &format!("[{}]:{}", address, DENDRITE_PORT), + "astring", )?; if *address != Ipv6Addr::LOCALHOST { let az_prefix = @@ -4074,9 +4086,10 @@ impl ServiceManager { for addr in Resolver::servers_from_subnet(az_prefix) { - smfh.addpropvalue( + smfh.addpropvalue_type_default_instance( "config/dns_server", &format!("{addr}"), + "astring", )?; } } @@ -4094,7 +4107,7 @@ impl ServiceManager { "rack_subnet" => %rack_subnet.net().addr(), ); - smfh.setprop( + smfh.setprop_default_instance( "config/rack-subnet", &rack_subnet.net().addr().to_string(), )?; @@ -4115,9 +4128,10 @@ impl ServiceManager { "*", )?; for address in &request.addresses { - smfh.addpropvalue_default_instance( + smfh.addpropvalue_type_default_instance( "config/address", &format!("[{}]:{}", address, LLDP_PORT), + "astring", )?; } smfh.refresh()?; @@ -4142,10 +4156,16 @@ impl ServiceManager { } SwitchService::Mgd => { info!(self.inner.log, "configuring mgd service"); - smfh.delpropvalue("config/dns_servers", "*")?; + smfh.delpropvalue_default_instance( + "config/dns_servers", + "*", + )?; if let Some(info) = self.inner.sled_info.get() { - smfh.setprop("config/rack_uuid", info.rack_id)?; - smfh.setprop( + smfh.setprop_default_instance( + "config/rack_uuid", + info.rack_id, + )?; + smfh.setprop_default_instance( "config/sled_uuid", info.config.sled_id, )?; @@ -4157,9 +4177,10 @@ impl ServiceManager { for addr in Resolver::servers_from_subnet(az_prefix) { - smfh.addpropvalue( + smfh.addpropvalue_type_default_instance( "config/dns_servers", &format!("{addr}"), + "astring", )?; } break; @@ -4173,16 +4194,29 @@ impl ServiceManager { } SwitchService::MgDdm { mode } => { info!(self.inner.log, "configuring mg-ddm service"); - smfh.delpropvalue("config/mode", "*")?; - smfh.addpropvalue("config/mode", &mode)?; + smfh.delpropvalue_default_instance( + "config/mode", + "*", + )?; + smfh.addpropvalue_type_default_instance( + "config/mode", + &mode, + "astring", + )?; if let Some(info) = self.inner.sled_info.get() { - smfh.setprop("config/rack_uuid", info.rack_id)?; - smfh.setprop( + smfh.setprop_default_instance( + "config/rack_uuid", + info.rack_id, + )?; + smfh.setprop_default_instance( "config/sled_uuid", info.config.sled_id, )?; } - smfh.delpropvalue("config/dns_servers", "*")?; + smfh.delpropvalue_default_instance( + "config/dns_servers", + "*", + )?; for address in &request.addresses { if *address != Ipv6Addr::LOCALHOST { let az_prefix = @@ -4190,9 +4224,10 @@ impl ServiceManager { for addr in Resolver::servers_from_subnet(az_prefix) { - smfh.addpropvalue( + smfh.addpropvalue_type_default_instance( "config/dns_servers", &format!("{addr}"), + "astring", )?; } break; diff --git a/sled-agent/src/smf_helper.rs b/sled-agent/src/smf_helper.rs index a7a14cdffd..b950c7b9b3 100644 --- a/sled-agent/src/smf_helper.rs +++ b/sled-agent/src/smf_helper.rs @@ -35,7 +35,11 @@ impl<'t> SmfHelper<'t> { SmfHelper { running_zone, _service_name, smf_name, default_smf_name } } - pub fn setprop(&self, prop: P, val: V) -> Result<(), Error> + pub fn setprop_default_instance( + &self, + prop: P, + val: V, + ) -> Result<(), Error> where P: ToString, V: ToString, @@ -44,7 +48,7 @@ impl<'t> SmfHelper<'t> { .run_cmd(&[ illumos_utils::zone::SVCCFG, "-s", - &self.smf_name, + &self.default_smf_name, "setprop", &format!("{}={}", prop.to_string(), val.to_string()), ]) @@ -83,35 +87,16 @@ impl<'t> SmfHelper<'t> { Ok(()) } - pub fn addpropvalue(&self, prop: P, val: V) -> Result<(), Error> - where - P: ToString, - V: ToString, - { - self.running_zone - .run_cmd(&[ - illumos_utils::zone::SVCCFG, - "-s", - &self.smf_name, - "addpropvalue", - &prop.to_string(), - &val.to_string(), - ]) - .map_err(|err| Error::ZoneCommand { - intent: format!("add {} smf property value", prop.to_string()), - err, - })?; - Ok(()) - } - - pub fn addpropvalue_default_instance( + pub fn addpropvalue_type_default_instance( &self, prop: P, val: V, + valtype: T, ) -> Result<(), Error> where P: ToString, V: ToString, + T: ToString, { self.running_zone .run_cmd(&[ @@ -120,6 +105,7 @@ impl<'t> SmfHelper<'t> { &self.default_smf_name, "addpropvalue", &prop.to_string(), + &format!("{}:", valtype.to_string()), &val.to_string(), ]) .map_err(|err| Error::ZoneCommand { @@ -180,27 +166,6 @@ impl<'t> SmfHelper<'t> { Ok(()) } - pub fn delpropvalue(&self, prop: P, val: V) -> Result<(), Error> - where - P: ToString, - V: ToString, - { - self.running_zone - .run_cmd(&[ - illumos_utils::zone::SVCCFG, - "-s", - &self.smf_name, - "delpropvalue", - &prop.to_string(), - &val.to_string(), - ]) - .map_err(|err| Error::ZoneCommand { - intent: format!("del {} smf property value", prop.to_string()), - err, - })?; - Ok(()) - } - pub fn delpropvalue_default_instance( &self, prop: P, @@ -210,7 +175,8 @@ impl<'t> SmfHelper<'t> { P: ToString, V: ToString, { - self.running_zone + match self + .running_zone .run_cmd(&[ illumos_utils::zone::SVCCFG, "-s", @@ -222,7 +188,17 @@ impl<'t> SmfHelper<'t> { .map_err(|err| Error::ZoneCommand { intent: format!("del {} smf property value", prop.to_string()), err, - })?; + }) { + Ok(_) => (), + Err(e) => { + // If a property already doesn't exist we don't need to + // return an error + if !e.to_string().contains("No such property") { + return Err(e); + } + } + }; + Ok(()) } From 77533189b3e68afd7deb9a8d936a951eba3b971a Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 17 Jun 2024 11:29:26 +1200 Subject: [PATCH 41/56] fix typo --- sled-agent/src/services.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index dd30b70814..867c826cce 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -2980,7 +2980,7 @@ impl ServiceManager { // Part of the process to ensure bootstrap address is to set up // an IPv6 address within the Global Zone. - // This means we cannot run bootsrap setup via a service running on + // This means we cannot run bootstrap setup via a service running on // the switch zone itself. if let Some((bootstrap_name, bootstrap_address)) = bootstrap_name_and_address.as_ref() From 86bef4dd7f3a095ac885e2907400497988265bca Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 17 Jun 2024 13:55:28 +1200 Subject: [PATCH 42/56] Remove unecessary clone --- illumos-utils/src/link.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/illumos-utils/src/link.rs b/illumos-utils/src/link.rs index 484c05b32e..871ba55e75 100644 --- a/illumos-utils/src/link.rs +++ b/illumos-utils/src/link.rs @@ -155,7 +155,6 @@ pub struct InvalidLinkKind(String); /// Note that the "ownership" of the VNIC is based on convention; /// another process in the global zone could also modify / destroy /// the VNIC while this object is alive. -#[derive(Clone)] pub struct Link { name: String, deleted: bool, From ebb6fce155f5e7b75bb19adaf6af85115c18db2c Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 17 Jun 2024 14:07:16 +1200 Subject: [PATCH 43/56] Clean up --- sled-agent/src/smf_helper.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sled-agent/src/smf_helper.rs b/sled-agent/src/smf_helper.rs index b950c7b9b3..230f146323 100644 --- a/sled-agent/src/smf_helper.rs +++ b/sled-agent/src/smf_helper.rs @@ -21,18 +21,16 @@ pub trait Service { pub struct SmfHelper<'t> { running_zone: &'t RunningZone, - _service_name: String, smf_name: String, default_smf_name: String, } impl<'t> SmfHelper<'t> { pub fn new(running_zone: &'t RunningZone, service: &impl Service) -> Self { - let _service_name = service.service_name(); let smf_name = service.smf_name(); let default_smf_name = format!("{}:default", smf_name); - SmfHelper { running_zone, _service_name, smf_name, default_smf_name } + SmfHelper { running_zone, smf_name, default_smf_name } } pub fn setprop_default_instance( From 70be78e6965e29949a8ff2267deaac6fef0b3113 Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 17 Jun 2024 14:59:19 +1200 Subject: [PATCH 44/56] Address comment --- sled-agent/src/services.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 867c826cce..36b825ddec 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1421,7 +1421,7 @@ impl ServiceManager { fn zone_network_setup_install( gw_addr: &Ipv6Addr, zone: &InstalledZone, - static_addrs: &Vec, + static_addrs: &[Ipv6Addr], ) -> Result { let datalink = zone.get_control_vnic_name(); let gateway = &gw_addr.to_string(); @@ -1604,7 +1604,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![listen_addr], + &[listen_addr], )?; let dns_service = Self::dns_install(info, None, &None).await?; @@ -1657,7 +1657,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![listen_addr], + &[listen_addr], )?; let dns_service = Self::dns_install(info, None, &None).await?; @@ -1720,7 +1720,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![crdb_listen_ip], + &[crdb_listen_ip], )?; let dns_service = Self::dns_install(info, None, &None).await?; @@ -1784,7 +1784,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![listen_addr], + &[listen_addr], )?; let dataset_name = DatasetName::new( @@ -1843,7 +1843,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![listen_addr], + &[listen_addr], )?; let config = PropertyGroupBuilder::new("config") @@ -1895,7 +1895,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![*underlay_address], + &[*underlay_address], )?; let oximeter_config = PropertyGroupBuilder::new("config") @@ -1936,7 +1936,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![*underlay_address], + &[*underlay_address], )?; // Like Nexus, we need to be reachable externally via // `dns_address` but we don't listen on that address @@ -2023,7 +2023,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![*underlay_address], + &[*underlay_address], )?; let is_boundary = matches!( @@ -2116,7 +2116,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( gz_address, &installed_zone, - &vec![*underlay_address], + &[*underlay_address], )?; // Internal DNS zones require a special route through @@ -2199,7 +2199,7 @@ impl ServiceManager { let nw_setup_service = Self::zone_network_setup_install( &info.underlay_address, &installed_zone, - &vec![*underlay_address], + &[*underlay_address], )?; // While Nexus will be reachable via `external_ip`, it From 45678c3199488b69483b53ad3fa73685ffce3b15 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 19 Jun 2024 10:37:03 +1200 Subject: [PATCH 45/56] Adding SP sim to property builder isn't necessary and is cleaner on a live deployment --- sled-agent/src/services.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 36b825ddec..528a06dfb9 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -2359,7 +2359,6 @@ impl ServiceManager { let mut mgd_service = ServiceBuilder::new("oxide/mgd"); let mut mg_ddm_service = ServiceBuilder::new("oxide/mg-ddm"); let mut uplink_service = ServiceBuilder::new("oxide/uplink"); - let mut sp_sim_service = ServiceBuilder::new("oxide/sp-sim"); let mut switch_zone_setup_config = PropertyGroupBuilder::new("config"); @@ -2423,9 +2422,6 @@ impl ServiceManager { self.inner.log, "Setting up Simulated SP service" ); - sp_sim_service = sp_sim_service.add_instance( - ServiceInstanceBuilder::new("default"), - ); } SwitchService::Wicketd { baseboard } => { info!(self.inner.log, "Setting up wicketd service"); @@ -2965,7 +2961,6 @@ impl ServiceManager { .add_service(pumpkind_service) .add_service(mgd_service) .add_service(mg_ddm_service) - .add_service(sp_sim_service) .add_service(uplink_service); profile .add_to_zone(&self.inner.log, &installed_zone) From 17c4931e21c1133b67d2506fa03ae13356ecbbbb Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 20 Jun 2024 20:48:20 +1200 Subject: [PATCH 46/56] Modify start for when underlay is not available yet --- sled-agent/src/services.rs | 107 ++++++++++++++++++++----------- zone-setup/src/bin/zone-setup.rs | 79 ++++++++++++++--------- 2 files changed, 116 insertions(+), 70 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 528a06dfb9..5b9809cc73 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1419,12 +1419,18 @@ impl ServiceManager { } fn zone_network_setup_install( - gw_addr: &Ipv6Addr, + gw_addr: Option<&Ipv6Addr>, zone: &InstalledZone, static_addrs: &[Ipv6Addr], ) -> Result { let datalink = zone.get_control_vnic_name(); - let gateway = &gw_addr.to_string(); + + // The switch zone is the only zone that will sometimes have an + // unknown underlay address at zone boot on the first scrimlet. + let gateway = match gw_addr { + Some(addr) => addr.to_string(), + None => "unknown".to_string(), + }; let mut config_builder = PropertyGroupBuilder::new("config"); config_builder = config_builder @@ -1602,7 +1608,7 @@ impl ServiceManager { let listen_port = &CLICKHOUSE_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + Some(&info.underlay_address), &installed_zone, &[listen_addr], )?; @@ -1655,7 +1661,7 @@ impl ServiceManager { let listen_port = &CLICKHOUSE_KEEPER_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + Some(&info.underlay_address), &installed_zone, &[listen_addr], )?; @@ -1718,7 +1724,7 @@ impl ServiceManager { .to_string(); let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + Some(&info.underlay_address), &installed_zone, &[crdb_listen_ip], )?; @@ -1782,7 +1788,7 @@ impl ServiceManager { let listen_port = &CRUCIBLE_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + Some(&info.underlay_address), &installed_zone, &[listen_addr], )?; @@ -1841,7 +1847,7 @@ impl ServiceManager { let listen_port = &CRUCIBLE_PANTRY_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + Some(&info.underlay_address), &installed_zone, &[listen_addr], )?; @@ -1893,7 +1899,7 @@ impl ServiceManager { ); let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + Some(&info.underlay_address), &installed_zone, &[*underlay_address], )?; @@ -1934,7 +1940,7 @@ impl ServiceManager { }; let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + Some(&info.underlay_address), &installed_zone, &[*underlay_address], )?; @@ -2021,7 +2027,7 @@ impl ServiceManager { }; let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + Some(&info.underlay_address), &installed_zone, &[*underlay_address], )?; @@ -2114,7 +2120,7 @@ impl ServiceManager { .. }) => { let nw_setup_service = Self::zone_network_setup_install( - gz_address, + Some(gz_address), &installed_zone, &[*underlay_address], )?; @@ -2197,7 +2203,7 @@ impl ServiceManager { }; let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + Some(&info.underlay_address), &installed_zone, &[*underlay_address], )?; @@ -2326,12 +2332,15 @@ impl ServiceManager { zone: SwitchZoneConfig { id, services, addresses }, .. }) => { - let Some(info) = self.inner.sled_info.get() else { - return Err(Error::SledAgentNotReady); + let info = self.inner.sled_info.get(); + + let gw_addr = match info { + Some(i) => Some(&i.underlay_address), + None => None, }; let nw_setup_service = Self::zone_network_setup_install( - &info.underlay_address, + gw_addr, &installed_zone, addresses, )?; @@ -2395,13 +2404,16 @@ impl ServiceManager { "id", "astring", &id.to_string(), - ) - .add_property( - "rack_id", - "astring", - &info.rack_id.to_string(), ); + if let Some(i) = info { + mgs_config = mgs_config.add_property( + "rack_id", + "astring", + &i.rack_id.to_string(), + ); + } + if let Some(address) = addresses.get(0) { // Don't use localhost twice if *address != Ipv6Addr::LOCALHOST { @@ -2446,11 +2458,7 @@ impl ServiceManager { }); }; - let rack_subnet = Ipv6Subnet::::new( - info.underlay_address, - ); - - let wicketd_config = + let mut wicketd_config = PropertyGroupBuilder::new("config") .add_property( "address", @@ -2481,13 +2489,20 @@ impl ServiceManager { "nexus-proxy-address", "astring", &format!("[::]:{WICKETD_NEXUS_PROXY_PORT}"), - ) - .add_property( - "rack-subnet", - "astring", - &rack_subnet.net().addr().to_string(), ); + if let Some(i) = info { + let rack_subnet = Ipv6Subnet::::new( + i.underlay_address, + ); + + wicketd_config = wicketd_config.add_property( + "rack-subnet", + "astring", + &rack_subnet.net().addr().to_string(), + ); + } + wicketd_service = wicketd_service.add_instance( ServiceInstanceBuilder::new("default") .add_property_group(wicketd_config), @@ -2509,17 +2524,21 @@ impl ServiceManager { "Setting up dendrite service" ); let mut dendrite_config = - PropertyGroupBuilder::new("config") + PropertyGroupBuilder::new("config"); + + if let Some(i) = info { + dendrite_config = dendrite_config .add_property( "sled_id", "astring", - &info.config.sled_id.to_string(), + &i.config.sled_id.to_string(), ) .add_property( "rack_id", "astring", - &info.rack_id.to_string(), + &i.rack_id.to_string(), ); + } for address in addresses { dendrite_config = dendrite_config.add_property( @@ -2802,17 +2821,21 @@ impl ServiceManager { info!(self.inner.log, "Setting up mgd service"); let mut mgd_config = - PropertyGroupBuilder::new("config") + PropertyGroupBuilder::new("config"); + + if let Some(i) = info { + mgd_config = mgd_config .add_property( "sled_uuid", "astring", - &info.config.sled_id.to_string(), + &i.config.sled_id.to_string(), ) .add_property( "rack_uuid", "astring", - &info.rack_id.to_string(), + &i.rack_id.to_string(), ); + } for address in addresses { if *address != Ipv6Addr::LOCALHOST { @@ -2842,17 +2865,23 @@ impl ServiceManager { let mut mg_ddm_config = PropertyGroupBuilder::new("config") .add_property("mode", "astring", mode) - .add_property("dendrite", "astring", "true") + .add_property( + "dendrite", "astring", "true", + ); + + if let Some(i) = info { + mg_ddm_config = mg_ddm_config .add_property( "sled_uuid", "astring", - &info.config.sled_id.to_string(), + &i.config.sled_id.to_string(), ) .add_property( "rack_uuid", "astring", - &info.rack_id.to_string(), + &i.rack_id.to_string(), ); + } for address in addresses { if *address != Ipv6Addr::LOCALHOST { diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 3b11295dda..06e01dd4cb 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -23,6 +23,7 @@ use std::io::Write; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::os::unix::fs::chown; use std::path::Path; +use std::str::FromStr; use uzers::{get_group_by_name, get_user_by_name}; use zone_setup::switch_zone_user::SwitchZoneUser; @@ -58,6 +59,16 @@ fn parse_ipv6(s: &str) -> anyhow::Result { s.parse().map_err(|_| anyhow!("ERROR: Invalid IPv6 address")) } +fn parse_gateway(s: &str) -> anyhow::Result> { + if s == "unknown" || s == "" { + return Ok(None); + }; + + let gw = Ipv6Addr::from_str(s) + .map_err(|_| anyhow!("ERROR: Invalid IPv6 address"))?; + Ok(Some(gw)) +} + fn parse_datalink(s: &str) -> anyhow::Result { if s == "unknown" { return Err(anyhow!("ERROR: Missing data link")); @@ -139,7 +150,7 @@ async fn do_run() -> Result<(), CmdError> { -g --gateway "gateway" ) .required(true) - .value_parser(parse_ipv6), + .value_parser(parse_gateway), ) .arg( Arg::new("static_addrs") @@ -584,7 +595,7 @@ async fn common_nw_set_up( .get_many::("static_addrs") .unwrap() .collect::>(); - let gateway: Ipv6Addr = *matches.get_one("gateway").unwrap(); + let gateway: Option = *matches.get_one("gateway").unwrap(); let zonename = zone::current().await.map_err(|err| { CmdError::Failure(anyhow!( "Could not determine local zone name: {}", @@ -627,35 +638,39 @@ async fn common_nw_set_up( } } - // NOTE: Ensuring default route with gateway must happen after peer agents have been initialized. - // Omicron zones will be able ensure a default route with gateway immediately, but the switch zone - // on the secondary scrimlet might need a few tries while it waits. - retry_notify( - retry_policy_local(), - || async { - info!(&log, "Ensuring there is a default route"; "gateway" => ?gateway); - Route::ensure_default_route_with_gateway(Gateway::Ipv6(gateway)) - .map_err(|err| { - match err { - ExecutionError::CommandFailure(ref e) => { - if e.stdout.contains("Network is unreachable") { - BackoffError::transient( - CmdError::Failure(anyhow!(err)), - ) - } else { - BackoffError::permanent( - CmdError::Failure(anyhow!(err)), - ) - } - } - _ => { - BackoffError::permanent( - CmdError::Failure(anyhow!(err)), - ) - } - } - }) - }, + match gateway { + // Only the switch zone will sometimes have an unknown underlay address at zone boot. + None => info!(&log, "Underlay is not available yet. Not ensuring there is a default route"), + Some(gw) => { + // Ensuring default route with gateway must happen after peer agents have been initialized. + // Omicron zones will be able ensure a default route with gateway immediately, but the + // switch zone on the secondary scrimlet might need a few tries while it waits. + retry_notify( + retry_policy_local(), + || async { + info!(&log, "Ensuring there is a default route"; "gateway" => ?gw); + Route::ensure_default_route_with_gateway(Gateway::Ipv6(gw)) + .map_err(|err| { + match err { + ExecutionError::CommandFailure(ref e) => { + if e.stdout.contains("Network is unreachable") { + BackoffError::transient( + CmdError::Failure(anyhow!(err)), + ) + } else { + BackoffError::permanent( + CmdError::Failure(anyhow!(err)), + ) + } + } + _ => { + BackoffError::permanent( + CmdError::Failure(anyhow!(err)), + ) + } + } + }) + }, |err, delay| { info!( &log, @@ -666,6 +681,8 @@ async fn common_nw_set_up( }, ) .await?; + } + } info!(&log, "Populating hosts file for zone"; "zonename" => ?zonename); let mut hosts_contents = String::from( From 2f146485d582dee1e832bb3046d54b670a426f0a Mon Sep 17 00:00:00 2001 From: karencfv Date: Tue, 25 Jun 2024 13:10:47 +1200 Subject: [PATCH 47/56] Add default route during second run of setting property values --- sled-agent/src/services.rs | 29 +++++++++++++++++++++++++++++ zone-setup/src/bin/zone-setup.rs | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 5b9809cc73..bd108e6007 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -4018,6 +4018,35 @@ impl ServiceManager { ); } + // When the request addresses have changed this means the underlay is + // available now as well. + if let Some(info) = self.inner.sled_info.get() { + info!( + self.inner.log, + "Ensuring there is a default route"; + "gateway" => ?info.underlay_address, + ); + match zone.add_default_route(info.underlay_address).map_err( + |err| Error::ZoneCommand { + intent: "Adding Route".to_string(), + err, + }, + ) { + Ok(_) => (), + Err(e) => { + if e.to_string().contains("entry exists") { + info!( + self.inner.log, + "Default route already exists"; + "gateway" => ?info.underlay_address, + ) + } else { + return Err(e); + } + } + }; + } + for service in &request.services { let smfh = SmfHelper::new(&zone, service); diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 06e01dd4cb..3ca525a2e7 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -667,8 +667,8 @@ async fn common_nw_set_up( BackoffError::permanent( CmdError::Failure(anyhow!(err)), ) - } } + } }) }, |err, delay| { From 9499512dd67d4daed2445083ffd5c8378278ce91 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 26 Jun 2024 14:54:05 +1200 Subject: [PATCH 48/56] Add some logging --- illumos-utils/src/running_zone.rs | 28 +++++--- illumos-utils/src/zone.rs | 102 ++++++++++++++++++++++++++++-- sled-agent/src/services.rs | 4 +- sled-hardware/src/underlay.rs | 2 +- zone-setup/src/bin/zone-setup.rs | 1 + 5 files changed, 122 insertions(+), 15 deletions(-) diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index b5e153d708..648027afe4 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -617,7 +617,7 @@ impl RunningZone { err, })?; let network = - Zones::ensure_address(Some(&self.inner.name), &addrobj, addrtype)?; + Zones::ensure_address(None, Some(&self.inner.name), &addrobj, addrtype)?; Ok(network) } @@ -626,7 +626,7 @@ impl RunningZone { &self, address: Ipv6Addr, ) -> Result<(), EnsureAddressError> { - info!(self.inner.log, "Adding bootstrap address"); + info!(self.inner.log, "DEBUG RunningZone::ensure_bootstrap_address: Adding bootstrap address"); let vnic = self.inner.bootstrap_vnic.as_ref().ok_or_else(|| { EnsureAddressError::MissingBootstrapVnic { address: address.to_string(), @@ -643,8 +643,13 @@ impl RunningZone { err, } })?; + info!(self.inner.log, "DEBUG RunningZone::ensure_bootstrap_address: Ensuring address"; + "zone" => ?self.inner.name, + "addrobj" => #?addrobj, + "addrtype" => #?addrtype, + ); let _ = - Zones::ensure_address(Some(&self.inner.name), &addrobj, addrtype)?; + Zones::ensure_address(Some(&self.inner.log), Some(&self.inner.name), &addrobj, addrtype)?; Ok(()) } @@ -672,7 +677,7 @@ impl RunningZone { let zone = Some(self.inner.name.as_ref()); if let IpAddr::V4(gateway) = port.gateway().ip() { let addr = - Zones::ensure_address(zone, &addrobj, AddressRequest::Dhcp)?; + Zones::ensure_address(None, zone, &addrobj, AddressRequest::Dhcp)?; // TODO-remove(#2931): OPTE's DHCP "server" returns the list of routes // to add via option 121 (Classless Static Route). The illumos DHCP // client currently does not support this option, so we add the routes @@ -700,7 +705,7 @@ impl RunningZone { } else { // If the port is using IPv6 addressing we still want it to use // DHCP(v6) which requires first creating a link-local address. - Zones::ensure_has_link_local_v6_address(zone, &addrobj).map_err( + Zones::ensure_has_link_local_v6_address(None, zone, &addrobj).map_err( |err| EnsureAddressError::LinkLocal { zone: self.inner.name.clone(), err, @@ -788,7 +793,7 @@ impl RunningZone { gz_bootstrap_addr: Ipv6Addr, zone_vnic_name: &str, ) -> Result<(), RunCommandError> { - self.run_cmd([ + let args = [ "/usr/sbin/route", "add", "-inet6", @@ -796,7 +801,14 @@ impl RunningZone { &gz_bootstrap_addr.to_string(), "-ifp", zone_vnic_name, - ])?; + ]; + info!(self.inner.log, "DEBUG RunningZone::add_bootstrap_route: Adding bootstrap route"; + "bootstrap_prefix" => #?bootstrap_prefix, + "gz_bootstrap_addr" => #?gz_bootstrap_addr, + "zone_vnic_name" => #?zone_vnic_name, + "cmd" => #?args, + ); + self.run_cmd(args)?; Ok(()) } @@ -845,7 +857,7 @@ impl RunningZone { let addrobj = AddrObject::new_control(&vnic_name).map_err(|err| { GetZoneError::AddrObject { name: zone_name.to_string(), err } })?; - Zones::ensure_address(Some(zone_name), &addrobj, addrtype).map_err( + Zones::ensure_address(None, Some(zone_name), &addrobj, addrtype).map_err( |err| GetZoneError::EnsureAddress { name: zone_name.to_string(), err, diff --git a/illumos-utils/src/zone.rs b/illumos-utils/src/zone.rs index 3f749fc352..92e2b161e2 100644 --- a/illumos-utils/src/zone.rs +++ b/illumos-utils/src/zone.rs @@ -497,31 +497,68 @@ impl Zones { /// If `None` is supplied, the address is queried from the Global Zone. #[allow(clippy::needless_lifetimes)] pub fn ensure_address<'a>( + log: Option<&'a Logger>, zone: Option<&'a str>, addrobj: &AddrObject, addrtype: AddressRequest, ) -> Result { |zone, addrobj, addrtype| -> Result { + if let Some(l) = log { + info!(l, "DEBUG Zones::ensure_address: Get IP address of interface"; + "zone" => #?zone, + "addrobj" => #?addrobj, + ); + } match Self::get_address_impl(zone, addrobj) { Ok(addr) => { if let AddressRequest::Static(expected_addr) = addrtype { // If the address is static, we need to validate that it // matches the value we asked for. if addr != expected_addr { + if let Some(log) = log { + info!(log, "DEBUG Zones::ensure_address: IP address of interface is not the expected one"; + "addr" => #?addr, + "expected_addr" => #?expected_addr, + ); + } // If the address doesn't match, try removing the old // value before using the new one. + if let Some(l) = log { + info!(l, "DEBUG Zones::ensure_address: Deleting old addrobj"; + "zone" => #?zone, + "addrobj" => #?addrobj, + ); + } Self::delete_address(zone, addrobj) .map_err(|e| anyhow!(e))?; + + if let Some(l) = log { + info!(l, "DEBUG Zones::ensure_address: Creating new addrobj"; + "zone" => #?zone, + "addrobj" => #?addrobj, + "addrtype" => #?addrtype, + ); + } return Self::create_address( - zone, addrobj, addrtype, + log, zone, addrobj, addrtype, ) .map_err(|e| anyhow!(e)); } } Ok(addr) } - Err(_) => Self::create_address(zone, addrobj, addrtype) - .map_err(|e| anyhow!(e)), + Err(err) => { + if let Some(log) = log { + info!(log, "DEBUG Zones::ensure_address: Creating new addrobj due to error when retrieving IP address of interface"; + "error" => #?err, + "zone" => #?zone, + "addrobj" => #?addrobj, + "addrtype" => #?addrtype, + ); + } + Self::create_address(log, zone, addrobj, addrtype) + .map_err(|e| anyhow!(e)) + } } }(zone, addrobj, addrtype) .map_err(|err| EnsureAddressError { @@ -641,6 +678,7 @@ impl Zones { // Does NOT check if the address already exists. #[allow(clippy::needless_lifetimes)] fn create_address_internal<'a>( + log: Option<&'a Logger>, zone: Option<&'a str>, addrobj: &AddrObject, addrtype: AddressRequest, @@ -671,6 +709,14 @@ impl Zones { args.push(addrobj.to_string()); let cmd = command.args(args); + if let Some(l) = log { + info!(l, "DEBUG Zones::create_address_internal: Attempt to create the requested address"; + "zone" => #?zone, + "addrobj" => #?addrobj, + "addrtype" => #?addrtype, + "cmd" => #?cmd, + ); + } execute(cmd)?; Ok(()) @@ -710,10 +756,18 @@ impl Zones { /// #[allow(clippy::needless_lifetimes)] pub fn ensure_has_link_local_v6_address<'a>( + log: Option<&'a Logger>, zone: Option<&'a str>, addrobj: &AddrObject, ) -> Result<(), crate::ExecutionError> { if let Ok(()) = Self::has_link_local_v6_address(zone, &addrobj) { + if let Some(l) = log { + info!(l, "DEBUG Zones::ensure_has_link_local_v6_address: addrobj Already has + corresponding link-local IPv6 address"; + "zone" => #?zone, + "addrobj" => #?addrobj, + ); + } return Ok(()); } @@ -734,6 +788,12 @@ impl Zones { let args = prefix.iter().chain(create_addr_args); let cmd = command.args(args); + if let Some(l) = log { + info!(l, "DEBUG Zones::ensure_has_link_local_v6_address: No link-local address was found, + attempt to make one."; + "cmd" => #?cmd, + ); + } execute(cmd)?; Ok(()) } @@ -754,6 +814,7 @@ impl Zones { let gz_link_local_addrobj = AddrObject::link_local(&link.0) .map_err(|err| anyhow!(err))?; Self::ensure_has_link_local_v6_address( + None, None, &gz_link_local_addrobj, ) @@ -766,6 +827,7 @@ impl Zones { // this sled itself are within the underlay or bootstrap prefix. // Anything else must be routed through Sidecar. Self::ensure_address( + None, None, &gz_link_local_addrobj .on_same_interface(name) @@ -806,6 +868,7 @@ impl Zones { // Creates an IP address within a Zone. #[allow(clippy::needless_lifetimes)] fn create_address<'a>( + log: Option<&'a Logger>, zone: Option<&'a str>, addrobj: &AddrObject, addrtype: AddressRequest, @@ -820,11 +883,35 @@ impl Zones { AddressRequest::Dhcp => {} AddressRequest::Static(addr) => { if addr.is_ipv6() { + if let Some(l) = log { + info!(l, "DEBUG Zones::create_address: Address is static and IPv6. + Also, we are allocating it in a non-global zone. + Doing prep work before allocating address"; + "zone" => #?zone, + "addr" => #?addr, + "addrtype" => #?addrtype, + ); + } + // Finally, actually ensure that the v6 address we want // exists within the zone. let link_local_addrobj = addrobj.link_local_on_same_interface()?; + if let Some(l) = log { + info!(l, "DEBUG Zones::create_address: Create a new addrobj + on the same interface with the IPv6 link-local name."; + "link_local_addrobj" => #?link_local_addrobj, + ); + } + if let Some(l) = log { + info!(l, "DEBUG Zones::create_address: Ensure a link-local + IPv6 exists with the name provided in addrobj"; + "zone" => #?zone, + "link_local_addrobj" => #?link_local_addrobj, + ); + } Self::ensure_has_link_local_v6_address( + log, Some(zone), &link_local_addrobj, )?; @@ -834,7 +921,14 @@ impl Zones { }; // Actually perform address allocation. - Self::create_address_internal(zone, addrobj, addrtype)?; + if let Some(l) = log { + info!(l, "DEBUG Zones::create_address: Actually perform address allocation."; + "zone" => #?zone, + "addrobj" => #?addrobj, + "addrtype" => #?addrtype, + ); + } + Self::create_address_internal(log, zone, addrobj, addrtype)?; Self::get_address_impl(zone, addrobj) } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index bd108e6007..6eae9e5bb5 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -3011,14 +3011,14 @@ impl ServiceManager { { info!( self.inner.log, - "Ensuring bootstrap address {} exists in {} zone", + "DEBUG ServiceManager::initialize_zone: Ensuring bootstrap address {} exists in {} zone", bootstrap_address.to_string(), &zone_type_str, ); running_zone.ensure_bootstrap_address(*bootstrap_address).await?; info!( self.inner.log, - "Forwarding bootstrap traffic via {} to {}", + "DEBUG ServiceManager::initialize_zone: Forwarding bootstrap traffic via {} to {}", bootstrap_name, self.inner.global_zone_bootstrap_link_local_address, ); diff --git a/sled-hardware/src/underlay.rs b/sled-hardware/src/underlay.rs index e0b40f443b..54e0dd09a6 100644 --- a/sled-hardware/src/underlay.rs +++ b/sled-hardware/src/underlay.rs @@ -92,7 +92,7 @@ pub fn ensure_links_have_global_zone_link_local_v6_addresses( for link in links { let addrobj = AddrObject::link_local(&link.0)?; - Zones::ensure_has_link_local_v6_address(None, &addrobj)?; + Zones::ensure_has_link_local_v6_address(None, None, &addrobj)?; addr_objs.push(addrobj); } diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 3ca525a2e7..ebc6ee3b67 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -308,6 +308,7 @@ async fn switch_zone_setup( info!(&log, "Ensuring link local links"; "links" => ?links); for link in &links { Zones::ensure_has_link_local_v6_address( + None, None, &AddrObject::new(link, IPV6_LINK_LOCAL_NAME).unwrap(), ) From 5a1d23e8d7358696e7e68271311ad796a678d9c5 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 26 Jun 2024 18:30:36 +1200 Subject: [PATCH 49/56] Set bootstrap address and link local in switch zone setup service --- illumos-utils/src/lib.rs | 2 +- illumos-utils/src/running_zone.rs | 40 +++++--- illumos-utils/src/zone.rs | 2 +- sled-agent/src/services.rs | 150 +++++++++++++++++++++++++---- smf/switch_zone_setup/manifest.xml | 4 +- zone-setup/src/bin/zone-setup.rs | 44 ++++++++- 6 files changed, 202 insertions(+), 40 deletions(-) diff --git a/illumos-utils/src/lib.rs b/illumos-utils/src/lib.rs index c5db9ed7de..03b4bfb5a7 100644 --- a/illumos-utils/src/lib.rs +++ b/illumos-utils/src/lib.rs @@ -37,7 +37,7 @@ pub struct CommandFailureInfo { command: String, status: std::process::ExitStatus, pub stdout: String, - stderr: String, + pub stderr: String, } impl std::fmt::Display for CommandFailureInfo { diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index 648027afe4..276c5723a7 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -45,7 +45,7 @@ pub enum ServiceError { pub struct RunCommandError { zone: String, #[source] - err: crate::ExecutionError, + pub err: crate::ExecutionError, } /// Errors returned from [`RunningZone::boot`]. @@ -616,8 +616,12 @@ impl RunningZone { zone: self.inner.name.clone(), err, })?; - let network = - Zones::ensure_address(None, Some(&self.inner.name), &addrobj, addrtype)?; + let network = Zones::ensure_address( + None, + Some(&self.inner.name), + &addrobj, + addrtype, + )?; Ok(network) } @@ -648,8 +652,12 @@ impl RunningZone { "addrobj" => #?addrobj, "addrtype" => #?addrtype, ); - let _ = - Zones::ensure_address(Some(&self.inner.log), Some(&self.inner.name), &addrobj, addrtype)?; + let _ = Zones::ensure_address( + Some(&self.inner.log), + Some(&self.inner.name), + &addrobj, + addrtype, + )?; Ok(()) } @@ -676,8 +684,12 @@ impl RunningZone { })?; let zone = Some(self.inner.name.as_ref()); if let IpAddr::V4(gateway) = port.gateway().ip() { - let addr = - Zones::ensure_address(None, zone, &addrobj, AddressRequest::Dhcp)?; + let addr = Zones::ensure_address( + None, + zone, + &addrobj, + AddressRequest::Dhcp, + )?; // TODO-remove(#2931): OPTE's DHCP "server" returns the list of routes // to add via option 121 (Classless Static Route). The illumos DHCP // client currently does not support this option, so we add the routes @@ -705,12 +717,11 @@ impl RunningZone { } else { // If the port is using IPv6 addressing we still want it to use // DHCP(v6) which requires first creating a link-local address. - Zones::ensure_has_link_local_v6_address(None, zone, &addrobj).map_err( - |err| EnsureAddressError::LinkLocal { + Zones::ensure_has_link_local_v6_address(None, zone, &addrobj) + .map_err(|err| EnsureAddressError::LinkLocal { zone: self.inner.name.clone(), err, - }, - )?; + })?; // Unlike DHCPv4, there's no blocking `ipadm` call we can // make as it just happens in the background. So we just poll @@ -857,12 +868,11 @@ impl RunningZone { let addrobj = AddrObject::new_control(&vnic_name).map_err(|err| { GetZoneError::AddrObject { name: zone_name.to_string(), err } })?; - Zones::ensure_address(None, Some(zone_name), &addrobj, addrtype).map_err( - |err| GetZoneError::EnsureAddress { + Zones::ensure_address(None, Some(zone_name), &addrobj, addrtype) + .map_err(|err| GetZoneError::EnsureAddress { name: zone_name.to_string(), err, - }, - )?; + })?; let control_vnic = vnic_allocator .wrap_existing(vnic_name) diff --git a/illumos-utils/src/zone.rs b/illumos-utils/src/zone.rs index 92e2b161e2..4ffa5012d4 100644 --- a/illumos-utils/src/zone.rs +++ b/illumos-utils/src/zone.rs @@ -677,7 +677,7 @@ impl Zones { // // Does NOT check if the address already exists. #[allow(clippy::needless_lifetimes)] - fn create_address_internal<'a>( + pub fn create_address_internal<'a>( log: Option<&'a Logger>, zone: Option<&'a str>, addrobj: &AddrObject, diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 6eae9e5bb5..69bc5ef1f6 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -57,6 +57,7 @@ use illumos_utils::running_zone::{ use illumos_utils::zfs::ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT; use illumos_utils::zone::AddressRequest; use illumos_utils::zpool::ZpoolName; +use illumos_utils::ExecutionError; use illumos_utils::{execute, PFEXEC}; use internal_dns::resolver::Resolver; use itertools::Itertools; @@ -83,7 +84,8 @@ use omicron_common::api::internal::shared::{ HostPortConfig, RackNetworkConfig, }; use omicron_common::backoff::{ - retry_notify, retry_policy_internal_service_aggressive, BackoffError, + retry_notify, retry_policy_internal_service_aggressive, retry_policy_local, + BackoffError, }; use omicron_common::ledger::{self, Ledger, Ledgerable}; use omicron_ddm_admin_client::{Client as DdmAdminClient, DdmError}; @@ -2385,6 +2387,38 @@ impl ServiceManager { } } + // TODO: Make this prettier + // let (bootstrap_vnic, bootstrap_name_and_address) = + // match self.bootstrap_address_needed(&request)? { + // Some((vnic, address)) => { + // let name = vnic.name().to_string(); + // (Some(vnic), Some((name, address))) + // } + // None => (None, None), + // }; + + if let Some((bootstrap_name, bootstrap_address)) = + bootstrap_name_and_address.as_ref() + { + // Add link_local for bootstrap address + switch_zone_setup_config = switch_zone_setup_config + .add_property( + "link_local_links", + "astring", + bootstrap_name, + ) + .add_property( + "bootstrap_addr", + "astring", + &format!("{bootstrap_address}"), + ) + .add_property( + "bootstrap_vnic", + "astring", + bootstrap_name, + ); + } + // Set properties for each service for service in services { match service { @@ -3006,33 +3040,109 @@ impl ServiceManager { // an IPv6 address within the Global Zone. // This means we cannot run bootstrap setup via a service running on // the switch zone itself. - if let Some((bootstrap_name, bootstrap_address)) = + if let Some((bootstrap_name, _bootstrap_address)) = bootstrap_name_and_address.as_ref() { - info!( - self.inner.log, - "DEBUG ServiceManager::initialize_zone: Ensuring bootstrap address {} exists in {} zone", - bootstrap_address.to_string(), - &zone_type_str, - ); - running_zone.ensure_bootstrap_address(*bootstrap_address).await?; - info!( - self.inner.log, - "DEBUG ServiceManager::initialize_zone: Forwarding bootstrap traffic via {} to {}", - bootstrap_name, - self.inner.global_zone_bootstrap_link_local_address, - ); - running_zone + // info!( + // self.inner.log, + // "DEBUG ServiceManager::initialize_zone: Ensuring bootstrap address {} exists in {} zone", + // bootstrap_address.to_string(), + // &zone_type_str, + // ); + // running_zone.ensure_bootstrap_address(*bootstrap_address).await?; + + retry_notify( + retry_policy_local(), + || async { + info!( + self.inner.log, + "DEBUG ServiceManager::initialize_zone: Forwarding bootstrap traffic via {} to {}", + bootstrap_name, + self.inner.global_zone_bootstrap_link_local_address, + ); + running_zone .add_bootstrap_route( BOOTSTRAP_PREFIX, self.inner.global_zone_bootstrap_link_local_address, bootstrap_name, ) - .map_err(|err| Error::ZoneCommand { - intent: "add bootstrap network route".to_string(), - err, - })?; + .map_err(|err| { + match err.err { + ExecutionError::CommandFailure(ref e) => { + if e.stderr.contains("no such interface") { + BackoffError::transient( + Error::ZoneCommand{ + intent: "add bootstrap network route".to_string(), + err, + } + ) + } else { + BackoffError::permanent( + Error::ZoneCommand{ + intent: "add bootstrap network route".to_string(), + err, + } + ) + } + } + _ => { + BackoffError::permanent( + Error::ZoneCommand{ + intent: "add bootstrap network route".to_string(), + err, + } + ) + } + } + }) + }, + |err, delay| { + info!( + &self.inner.log, + "Cannot forward bootstrap traffic via {} to {} yet (retrying in {:?})", + bootstrap_name, + self.inner.global_zone_bootstrap_link_local_address, + delay; + "error" => ?err + ); + }, + ) + .await?; + + // .map_err(|err| Error::ZoneCommand { + // intent: "add bootstrap network route".to_string(), + // err, + // })?; } + + // if let Some((bootstrap_name, _bootstrap_address)) = + // bootstrap_name_and_address.as_ref() + // { + // info!( + // self.inner.log, + // "DEBUG ServiceManager::initialize_zone: Ensuring bootstrap address {} exists in {} zone", + // bootstrap_address.to_string(), + // &zone_type_str, + // ); + // running_zone.ensure_bootstrap_address(*bootstrap_address).await?; + // info!( + // self.inner.log, + // "DEBUG ServiceManager::initialize_zone: Forwarding bootstrap traffic via {} to {}", + // bootstrap_name, + // self.inner.global_zone_bootstrap_link_local_address, + // ); + // running_zone + // .add_bootstrap_route( + // BOOTSTRAP_PREFIX, + // self.inner.global_zone_bootstrap_link_local_address, + // bootstrap_name, + // ) + // .map_err(|err| Error::ZoneCommand { + // intent: "add bootstrap network route".to_string(), + // err, + // })?; + // } + Ok(running_zone) } diff --git a/smf/switch_zone_setup/manifest.xml b/smf/switch_zone_setup/manifest.xml index 560c0a36f0..ce1ad87c82 100644 --- a/smf/switch_zone_setup/manifest.xml +++ b/smf/switch_zone_setup/manifest.xml @@ -12,7 +12,7 @@ @@ -24,6 +24,8 @@ + + diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index ebc6ee3b67..bc94e1483d 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -10,7 +10,7 @@ use illumos_utils::addrobj::{AddrObject, IPV6_LINK_LOCAL_NAME}; use illumos_utils::ipadm::Ipadm; use illumos_utils::route::{Gateway, Route}; use illumos_utils::svcadm::Svcadm; -use illumos_utils::zone::Zones; +use illumos_utils::zone::{AddressRequest, Zones}; use illumos_utils::ExecutionError; use omicron_common::backoff::{retry_notify, retry_policy_local, BackoffError}; use omicron_common::cmd::fatal; @@ -205,6 +205,19 @@ async fn do_run() -> Result<(), CmdError> { .required(true) .value_parser(parse_baseboard_info), ) + .arg( + arg!( + -a --bootstrap_addr "bootstrap_addr" + ) + .required(true) + .value_parser(parse_ipv6), + ) + .arg( + arg!( + -v --bootstrap_vnic "bootstrap_vnic" + ) + .required(true), + ) .arg( Arg::new("link_local_links") .short('l') @@ -270,6 +283,8 @@ async fn switch_zone_setup( ) -> Result<(), CmdError> { let file: &String = matches.get_one("baseboard_file").unwrap(); let info: &String = matches.get_one("baseboard_info").unwrap(); + let bootstrap_addr: &Ipv6Addr = matches.get_one("bootstrap_addr").unwrap(); + let bootstrap_vnic: &String = matches.get_one("bootstrap_vnic").unwrap(); let links = if let Some(l) = matches.get_many::("link_local_links") { Some(l.collect::>()) @@ -308,7 +323,7 @@ async fn switch_zone_setup( info!(&log, "Ensuring link local links"; "links" => ?links); for link in &links { Zones::ensure_has_link_local_v6_address( - None, + Some(&log), None, &AddrObject::new(link, IPV6_LINK_LOCAL_NAME).unwrap(), ) @@ -324,6 +339,31 @@ async fn switch_zone_setup( info!(&log, "No link local links to be configured"); }; + info!(&log, "Ensuring bootstrap address exists in zone"; + "bootstrap address" => ?bootstrap_addr, "bootstrap vnic" => ?bootstrap_vnic); + let addrtype = + AddressRequest::new_static(std::net::IpAddr::V6(*bootstrap_addr), None); + let addrobj_name = "bootstrap6"; + let addrobj = + AddrObject::new(&bootstrap_vnic, addrobj_name).map_err(|err| { + CmdError::Failure(anyhow!( + "Could not create new addrobj {:?}: {}", + addrobj_name, + err + )) + })?; + + let _ = + Zones::create_address_internal(Some(&log), None, &addrobj, addrtype) + .map_err(|err| { + CmdError::Failure(anyhow!( + "Could not create bootstrap address {} {:?}: {}", + addrobj, + addrtype, + err + )) + })?; + Ok(()) } From e3d40fa7c137bc235403815f00374daec20f2d42 Mon Sep 17 00:00:00 2001 From: karencfv Date: Wed, 26 Jun 2024 20:52:55 +1200 Subject: [PATCH 50/56] Clean up --- illumos-utils/src/running_zone.rs | 50 +++------ illumos-utils/src/zone.rs | 102 +----------------- sled-agent/src/services.rs | 165 ++++++++++-------------------- sled-hardware/src/underlay.rs | 2 +- zone-setup/src/bin/zone-setup.rs | 21 ++-- 5 files changed, 83 insertions(+), 257 deletions(-) diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index 276c5723a7..c4e68e0c50 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -616,12 +616,8 @@ impl RunningZone { zone: self.inner.name.clone(), err, })?; - let network = Zones::ensure_address( - None, - Some(&self.inner.name), - &addrobj, - addrtype, - )?; + let network = + Zones::ensure_address(Some(&self.inner.name), &addrobj, addrtype)?; Ok(network) } @@ -630,7 +626,6 @@ impl RunningZone { &self, address: Ipv6Addr, ) -> Result<(), EnsureAddressError> { - info!(self.inner.log, "DEBUG RunningZone::ensure_bootstrap_address: Adding bootstrap address"); let vnic = self.inner.bootstrap_vnic.as_ref().ok_or_else(|| { EnsureAddressError::MissingBootstrapVnic { address: address.to_string(), @@ -647,17 +642,8 @@ impl RunningZone { err, } })?; - info!(self.inner.log, "DEBUG RunningZone::ensure_bootstrap_address: Ensuring address"; - "zone" => ?self.inner.name, - "addrobj" => #?addrobj, - "addrtype" => #?addrtype, - ); - let _ = Zones::ensure_address( - Some(&self.inner.log), - Some(&self.inner.name), - &addrobj, - addrtype, - )?; + let _ = + Zones::ensure_address(Some(&self.inner.name), &addrobj, addrtype)?; Ok(()) } @@ -684,12 +670,8 @@ impl RunningZone { })?; let zone = Some(self.inner.name.as_ref()); if let IpAddr::V4(gateway) = port.gateway().ip() { - let addr = Zones::ensure_address( - None, - zone, - &addrobj, - AddressRequest::Dhcp, - )?; + let addr = + Zones::ensure_address(zone, &addrobj, AddressRequest::Dhcp)?; // TODO-remove(#2931): OPTE's DHCP "server" returns the list of routes // to add via option 121 (Classless Static Route). The illumos DHCP // client currently does not support this option, so we add the routes @@ -717,11 +699,12 @@ impl RunningZone { } else { // If the port is using IPv6 addressing we still want it to use // DHCP(v6) which requires first creating a link-local address. - Zones::ensure_has_link_local_v6_address(None, zone, &addrobj) - .map_err(|err| EnsureAddressError::LinkLocal { + Zones::ensure_has_link_local_v6_address(zone, &addrobj).map_err( + |err| EnsureAddressError::LinkLocal { zone: self.inner.name.clone(), err, - })?; + }, + )?; // Unlike DHCPv4, there's no blocking `ipadm` call we can // make as it just happens in the background. So we just poll @@ -813,12 +796,6 @@ impl RunningZone { "-ifp", zone_vnic_name, ]; - info!(self.inner.log, "DEBUG RunningZone::add_bootstrap_route: Adding bootstrap route"; - "bootstrap_prefix" => #?bootstrap_prefix, - "gz_bootstrap_addr" => #?gz_bootstrap_addr, - "zone_vnic_name" => #?zone_vnic_name, - "cmd" => #?args, - ); self.run_cmd(args)?; Ok(()) } @@ -868,11 +845,12 @@ impl RunningZone { let addrobj = AddrObject::new_control(&vnic_name).map_err(|err| { GetZoneError::AddrObject { name: zone_name.to_string(), err } })?; - Zones::ensure_address(None, Some(zone_name), &addrobj, addrtype) - .map_err(|err| GetZoneError::EnsureAddress { + Zones::ensure_address(Some(zone_name), &addrobj, addrtype).map_err( + |err| GetZoneError::EnsureAddress { name: zone_name.to_string(), err, - })?; + }, + )?; let control_vnic = vnic_allocator .wrap_existing(vnic_name) diff --git a/illumos-utils/src/zone.rs b/illumos-utils/src/zone.rs index 4ffa5012d4..deda449a3e 100644 --- a/illumos-utils/src/zone.rs +++ b/illumos-utils/src/zone.rs @@ -497,68 +497,31 @@ impl Zones { /// If `None` is supplied, the address is queried from the Global Zone. #[allow(clippy::needless_lifetimes)] pub fn ensure_address<'a>( - log: Option<&'a Logger>, zone: Option<&'a str>, addrobj: &AddrObject, addrtype: AddressRequest, ) -> Result { |zone, addrobj, addrtype| -> Result { - if let Some(l) = log { - info!(l, "DEBUG Zones::ensure_address: Get IP address of interface"; - "zone" => #?zone, - "addrobj" => #?addrobj, - ); - } match Self::get_address_impl(zone, addrobj) { Ok(addr) => { if let AddressRequest::Static(expected_addr) = addrtype { // If the address is static, we need to validate that it // matches the value we asked for. if addr != expected_addr { - if let Some(log) = log { - info!(log, "DEBUG Zones::ensure_address: IP address of interface is not the expected one"; - "addr" => #?addr, - "expected_addr" => #?expected_addr, - ); - } // If the address doesn't match, try removing the old // value before using the new one. - if let Some(l) = log { - info!(l, "DEBUG Zones::ensure_address: Deleting old addrobj"; - "zone" => #?zone, - "addrobj" => #?addrobj, - ); - } Self::delete_address(zone, addrobj) .map_err(|e| anyhow!(e))?; - - if let Some(l) = log { - info!(l, "DEBUG Zones::ensure_address: Creating new addrobj"; - "zone" => #?zone, - "addrobj" => #?addrobj, - "addrtype" => #?addrtype, - ); - } return Self::create_address( - log, zone, addrobj, addrtype, + zone, addrobj, addrtype, ) .map_err(|e| anyhow!(e)); } } Ok(addr) } - Err(err) => { - if let Some(log) = log { - info!(log, "DEBUG Zones::ensure_address: Creating new addrobj due to error when retrieving IP address of interface"; - "error" => #?err, - "zone" => #?zone, - "addrobj" => #?addrobj, - "addrtype" => #?addrtype, - ); - } - Self::create_address(log, zone, addrobj, addrtype) - .map_err(|e| anyhow!(e)) - } + Err(_) => Self::create_address(zone, addrobj, addrtype) + .map_err(|e| anyhow!(e)), } }(zone, addrobj, addrtype) .map_err(|err| EnsureAddressError { @@ -678,7 +641,6 @@ impl Zones { // Does NOT check if the address already exists. #[allow(clippy::needless_lifetimes)] pub fn create_address_internal<'a>( - log: Option<&'a Logger>, zone: Option<&'a str>, addrobj: &AddrObject, addrtype: AddressRequest, @@ -709,14 +671,6 @@ impl Zones { args.push(addrobj.to_string()); let cmd = command.args(args); - if let Some(l) = log { - info!(l, "DEBUG Zones::create_address_internal: Attempt to create the requested address"; - "zone" => #?zone, - "addrobj" => #?addrobj, - "addrtype" => #?addrtype, - "cmd" => #?cmd, - ); - } execute(cmd)?; Ok(()) @@ -756,18 +710,10 @@ impl Zones { /// #[allow(clippy::needless_lifetimes)] pub fn ensure_has_link_local_v6_address<'a>( - log: Option<&'a Logger>, zone: Option<&'a str>, addrobj: &AddrObject, ) -> Result<(), crate::ExecutionError> { if let Ok(()) = Self::has_link_local_v6_address(zone, &addrobj) { - if let Some(l) = log { - info!(l, "DEBUG Zones::ensure_has_link_local_v6_address: addrobj Already has - corresponding link-local IPv6 address"; - "zone" => #?zone, - "addrobj" => #?addrobj, - ); - } return Ok(()); } @@ -788,12 +734,6 @@ impl Zones { let args = prefix.iter().chain(create_addr_args); let cmd = command.args(args); - if let Some(l) = log { - info!(l, "DEBUG Zones::ensure_has_link_local_v6_address: No link-local address was found, - attempt to make one."; - "cmd" => #?cmd, - ); - } execute(cmd)?; Ok(()) } @@ -814,7 +754,6 @@ impl Zones { let gz_link_local_addrobj = AddrObject::link_local(&link.0) .map_err(|err| anyhow!(err))?; Self::ensure_has_link_local_v6_address( - None, None, &gz_link_local_addrobj, ) @@ -827,7 +766,6 @@ impl Zones { // this sled itself are within the underlay or bootstrap prefix. // Anything else must be routed through Sidecar. Self::ensure_address( - None, None, &gz_link_local_addrobj .on_same_interface(name) @@ -868,7 +806,6 @@ impl Zones { // Creates an IP address within a Zone. #[allow(clippy::needless_lifetimes)] fn create_address<'a>( - log: Option<&'a Logger>, zone: Option<&'a str>, addrobj: &AddrObject, addrtype: AddressRequest, @@ -883,35 +820,11 @@ impl Zones { AddressRequest::Dhcp => {} AddressRequest::Static(addr) => { if addr.is_ipv6() { - if let Some(l) = log { - info!(l, "DEBUG Zones::create_address: Address is static and IPv6. - Also, we are allocating it in a non-global zone. - Doing prep work before allocating address"; - "zone" => #?zone, - "addr" => #?addr, - "addrtype" => #?addrtype, - ); - } - // Finally, actually ensure that the v6 address we want // exists within the zone. let link_local_addrobj = addrobj.link_local_on_same_interface()?; - if let Some(l) = log { - info!(l, "DEBUG Zones::create_address: Create a new addrobj - on the same interface with the IPv6 link-local name."; - "link_local_addrobj" => #?link_local_addrobj, - ); - } - if let Some(l) = log { - info!(l, "DEBUG Zones::create_address: Ensure a link-local - IPv6 exists with the name provided in addrobj"; - "zone" => #?zone, - "link_local_addrobj" => #?link_local_addrobj, - ); - } Self::ensure_has_link_local_v6_address( - log, Some(zone), &link_local_addrobj, )?; @@ -921,14 +834,7 @@ impl Zones { }; // Actually perform address allocation. - if let Some(l) = log { - info!(l, "DEBUG Zones::create_address: Actually perform address allocation."; - "zone" => #?zone, - "addrobj" => #?addrobj, - "addrtype" => #?addrtype, - ); - } - Self::create_address_internal(log, zone, addrobj, addrtype)?; + Self::create_address_internal(zone, addrobj, addrtype)?; Self::get_address_impl(zone, addrobj) } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 69bc5ef1f6..8c901f0be0 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -2387,20 +2387,9 @@ impl ServiceManager { } } - // TODO: Make this prettier - // let (bootstrap_vnic, bootstrap_name_and_address) = - // match self.bootstrap_address_needed(&request)? { - // Some((vnic, address)) => { - // let name = vnic.name().to_string(); - // (Some(vnic), Some((name, address))) - // } - // None => (None, None), - // }; - if let Some((bootstrap_name, bootstrap_address)) = bootstrap_name_and_address.as_ref() { - // Add link_local for bootstrap address switch_zone_setup_config = switch_zone_setup_config .add_property( "link_local_links", @@ -3036,113 +3025,67 @@ impl ServiceManager { let running_zone = RunningZone::boot(installed_zone).await?; - // Part of the process to ensure bootstrap address is to set up - // an IPv6 address within the Global Zone. - // This means we cannot run bootstrap setup via a service running on - // the switch zone itself. - if let Some((bootstrap_name, _bootstrap_address)) = - bootstrap_name_and_address.as_ref() - { - // info!( - // self.inner.log, - // "DEBUG ServiceManager::initialize_zone: Ensuring bootstrap address {} exists in {} zone", - // bootstrap_address.to_string(), - // &zone_type_str, - // ); - // running_zone.ensure_bootstrap_address(*bootstrap_address).await?; - + if let Some((bootstrap_name, _)) = bootstrap_name_and_address.as_ref() { + // Forwarding bootstrap traffic requires the bootstrap interface to exist. + // We need to wait for it. retry_notify( - retry_policy_local(), - || async { - info!( - self.inner.log, - "DEBUG ServiceManager::initialize_zone: Forwarding bootstrap traffic via {} to {}", - bootstrap_name, - self.inner.global_zone_bootstrap_link_local_address, - ); - running_zone - .add_bootstrap_route( - BOOTSTRAP_PREFIX, - self.inner.global_zone_bootstrap_link_local_address, - bootstrap_name, - ) - .map_err(|err| { - match err.err { - ExecutionError::CommandFailure(ref e) => { - if e.stderr.contains("no such interface") { - BackoffError::transient( - Error::ZoneCommand{ - intent: "add bootstrap network route".to_string(), - err, - } - ) - } else { + retry_policy_local(), + || async { + info!( + self.inner.log, + "Forwarding bootstrap traffic via {} to {}", + bootstrap_name, + self.inner.global_zone_bootstrap_link_local_address, + ); + running_zone + .add_bootstrap_route( + BOOTSTRAP_PREFIX, + self.inner.global_zone_bootstrap_link_local_address, + bootstrap_name, + ) + .map_err(|err| { + match err.err { + ExecutionError::CommandFailure(ref e) => { + if e.stderr.contains("no such interface") { + BackoffError::transient( + Error::ZoneCommand{ + intent: "add bootstrap network route".to_string(), + err, + } + ) + } else { + BackoffError::permanent( + Error::ZoneCommand{ + intent: "add bootstrap network route".to_string(), + err, + } + ) + } + } + _ => { BackoffError::permanent( Error::ZoneCommand{ - intent: "add bootstrap network route".to_string(), - err, - } + intent: "add bootstrap network route".to_string(), + err, + } ) } } - _ => { - BackoffError::permanent( - Error::ZoneCommand{ - intent: "add bootstrap network route".to_string(), - err, - } - ) - } - } - }) - }, - |err, delay| { - info!( - &self.inner.log, - "Cannot forward bootstrap traffic via {} to {} yet (retrying in {:?})", - bootstrap_name, - self.inner.global_zone_bootstrap_link_local_address, - delay; - "error" => ?err - ); - }, - ) - .await?; - - // .map_err(|err| Error::ZoneCommand { - // intent: "add bootstrap network route".to_string(), - // err, - // })?; + }) + }, + |err, delay| { + info!( + &self.inner.log, + "Cannot forward bootstrap traffic via {} to {} yet (retrying in {:?})", + bootstrap_name, + self.inner.global_zone_bootstrap_link_local_address, + delay; + "error" => ?err + ); + }, + ) + .await?; } - - // if let Some((bootstrap_name, _bootstrap_address)) = - // bootstrap_name_and_address.as_ref() - // { - // info!( - // self.inner.log, - // "DEBUG ServiceManager::initialize_zone: Ensuring bootstrap address {} exists in {} zone", - // bootstrap_address.to_string(), - // &zone_type_str, - // ); - // running_zone.ensure_bootstrap_address(*bootstrap_address).await?; - // info!( - // self.inner.log, - // "DEBUG ServiceManager::initialize_zone: Forwarding bootstrap traffic via {} to {}", - // bootstrap_name, - // self.inner.global_zone_bootstrap_link_local_address, - // ); - // running_zone - // .add_bootstrap_route( - // BOOTSTRAP_PREFIX, - // self.inner.global_zone_bootstrap_link_local_address, - // bootstrap_name, - // ) - // .map_err(|err| Error::ZoneCommand { - // intent: "add bootstrap network route".to_string(), - // err, - // })?; - // } - Ok(running_zone) } diff --git a/sled-hardware/src/underlay.rs b/sled-hardware/src/underlay.rs index 54e0dd09a6..e0b40f443b 100644 --- a/sled-hardware/src/underlay.rs +++ b/sled-hardware/src/underlay.rs @@ -92,7 +92,7 @@ pub fn ensure_links_have_global_zone_link_local_v6_addresses( for link in links { let addrobj = AddrObject::link_local(&link.0)?; - Zones::ensure_has_link_local_v6_address(None, None, &addrobj)?; + Zones::ensure_has_link_local_v6_address(None, &addrobj)?; addr_objs.push(addrobj); } diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index bc94e1483d..6e916bae76 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -323,7 +323,6 @@ async fn switch_zone_setup( info!(&log, "Ensuring link local links"; "links" => ?links); for link in &links { Zones::ensure_has_link_local_v6_address( - Some(&log), None, &AddrObject::new(link, IPV6_LINK_LOCAL_NAME).unwrap(), ) @@ -353,16 +352,16 @@ async fn switch_zone_setup( )) })?; - let _ = - Zones::create_address_internal(Some(&log), None, &addrobj, addrtype) - .map_err(|err| { - CmdError::Failure(anyhow!( - "Could not create bootstrap address {} {:?}: {}", - addrobj, - addrtype, - err - )) - })?; + let _ = Zones::create_address_internal(None, &addrobj, addrtype).map_err( + |err| { + CmdError::Failure(anyhow!( + "Could not create bootstrap address {} {:?}: {}", + addrobj, + addrtype, + err + )) + }, + )?; Ok(()) } From ab6ddf376ce9af6725ee9366188e837af18fc4a2 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 27 Jun 2024 12:22:03 +1200 Subject: [PATCH 51/56] Include forwarding bootstrap traffic to switch zone start up service --- illumos-utils/src/route.rs | 19 ++++ sled-agent/src/services.rs | 138 +++++++++++++++-------------- smf/switch_zone_setup/manifest.xml | 3 +- zone-setup/src/bin/zone-setup.rs | 22 +++++ 4 files changed, 116 insertions(+), 66 deletions(-) diff --git a/illumos-utils/src/route.rs b/illumos-utils/src/route.rs index ceff2b3d9e..12f74bfd78 100644 --- a/illumos-utils/src/route.rs +++ b/illumos-utils/src/route.rs @@ -107,4 +107,23 @@ impl Route { }; Ok(()) } + + pub fn add_bootstrap_route( + bootstrap_prefix: u16, + gz_bootstrap_addr: Ipv6Addr, + zone_vnic_name: &str, + ) -> Result<(), ExecutionError> { + let mut cmd = std::process::Command::new(PFEXEC); + let cmd = cmd.args(&[ + ROUTE, + "add", + "-inet6", + &format!("{bootstrap_prefix:x}::/16"), + &gz_bootstrap_addr.to_string(), + "-ifp", + zone_vnic_name, + ]); + execute(cmd)?; + Ok(()) + } } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 8c901f0be0..b887d2b67f 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -57,7 +57,7 @@ use illumos_utils::running_zone::{ use illumos_utils::zfs::ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT; use illumos_utils::zone::AddressRequest; use illumos_utils::zpool::ZpoolName; -use illumos_utils::ExecutionError; +// use illumos_utils::ExecutionError; use illumos_utils::{execute, PFEXEC}; use internal_dns::resolver::Resolver; use itertools::Itertools; @@ -84,7 +84,7 @@ use omicron_common::api::internal::shared::{ HostPortConfig, RackNetworkConfig, }; use omicron_common::backoff::{ - retry_notify, retry_policy_internal_service_aggressive, retry_policy_local, + retry_notify, retry_policy_internal_service_aggressive, //retry_policy_local, BackoffError, }; use omicron_common::ledger::{self, Ledger, Ledgerable}; @@ -94,7 +94,7 @@ use rand::prelude::SliceRandom; use sled_hardware::is_gimlet; use sled_hardware::underlay; use sled_hardware::SledMode; -use sled_hardware_types::underlay::BOOTSTRAP_PREFIX; +//use sled_hardware_types::underlay::BOOTSTRAP_PREFIX; use sled_hardware_types::Baseboard; use sled_storage::config::MountConfig; use sled_storage::dataset::{ @@ -2372,7 +2372,15 @@ impl ServiceManager { let mut uplink_service = ServiceBuilder::new("oxide/uplink"); let mut switch_zone_setup_config = - PropertyGroupBuilder::new("config"); + PropertyGroupBuilder::new("config") + .add_property( + "gz_local_link_addr", + "astring", + &format!( + "{}", + self.inner.global_zone_bootstrap_link_local_address + ), + ); for (link, needs_link_local) in installed_zone.links().iter().zip(links_need_link_local) @@ -3025,67 +3033,67 @@ impl ServiceManager { let running_zone = RunningZone::boot(installed_zone).await?; - if let Some((bootstrap_name, _)) = bootstrap_name_and_address.as_ref() { - // Forwarding bootstrap traffic requires the bootstrap interface to exist. - // We need to wait for it. - retry_notify( - retry_policy_local(), - || async { - info!( - self.inner.log, - "Forwarding bootstrap traffic via {} to {}", - bootstrap_name, - self.inner.global_zone_bootstrap_link_local_address, - ); - running_zone - .add_bootstrap_route( - BOOTSTRAP_PREFIX, - self.inner.global_zone_bootstrap_link_local_address, - bootstrap_name, - ) - .map_err(|err| { - match err.err { - ExecutionError::CommandFailure(ref e) => { - if e.stderr.contains("no such interface") { - BackoffError::transient( - Error::ZoneCommand{ - intent: "add bootstrap network route".to_string(), - err, - } - ) - } else { - BackoffError::permanent( - Error::ZoneCommand{ - intent: "add bootstrap network route".to_string(), - err, - } - ) - } - } - _ => { - BackoffError::permanent( - Error::ZoneCommand{ - intent: "add bootstrap network route".to_string(), - err, - } - ) - } - } - }) - }, - |err, delay| { - info!( - &self.inner.log, - "Cannot forward bootstrap traffic via {} to {} yet (retrying in {:?})", - bootstrap_name, - self.inner.global_zone_bootstrap_link_local_address, - delay; - "error" => ?err - ); - }, - ) - .await?; - } + // if let Some((bootstrap_name, _)) = bootstrap_name_and_address.as_ref() { + // // Forwarding bootstrap traffic requires the bootstrap interface to exist. + // // We need to wait for it. + // retry_notify( + // retry_policy_local(), + // || async { + // info!( + // self.inner.log, + // "Forwarding bootstrap traffic via {} to {}", + // bootstrap_name, + // self.inner.global_zone_bootstrap_link_local_address, + // ); + // running_zone + // .add_bootstrap_route( + // BOOTSTRAP_PREFIX, + // self.inner.global_zone_bootstrap_link_local_address, + // bootstrap_name, + // ) + // .map_err(|err| { + // match err.err { + // ExecutionError::CommandFailure(ref e) => { + // if e.stderr.contains("no such interface") { + // BackoffError::transient( + // Error::ZoneCommand{ + // intent: "add bootstrap network route".to_string(), + // err, + // } + // ) + // } else { + // BackoffError::permanent( + // Error::ZoneCommand{ + // intent: "add bootstrap network route".to_string(), + // err, + // } + // ) + // } + // } + // _ => { + // BackoffError::permanent( + // Error::ZoneCommand{ + // intent: "add bootstrap network route".to_string(), + // err, + // } + // ) + // } + // } + // }) + // }, + // |err, delay| { + // info!( + // &self.inner.log, + // "Cannot forward bootstrap traffic via {} to {} yet (retrying in {:?})", + // bootstrap_name, + // self.inner.global_zone_bootstrap_link_local_address, + // delay; + // "error" => ?err + // ); + // }, + // ) + // .await?; + // } Ok(running_zone) } diff --git a/smf/switch_zone_setup/manifest.xml b/smf/switch_zone_setup/manifest.xml index ce1ad87c82..52170a0316 100644 --- a/smf/switch_zone_setup/manifest.xml +++ b/smf/switch_zone_setup/manifest.xml @@ -12,7 +12,7 @@ @@ -26,6 +26,7 @@ + diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 6e916bae76..6dd911c3c1 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -17,6 +17,7 @@ use omicron_common::cmd::fatal; use omicron_common::cmd::CmdError; use omicron_sled_agent::services::SWITCH_ZONE_BASEBOARD_FILE; use serde_json::Value; +use sled_hardware_types::underlay::BOOTSTRAP_PREFIX; use slog::{info, Logger}; use std::fs::{metadata, read_to_string, set_permissions, write, OpenOptions}; use std::io::Write; @@ -218,6 +219,13 @@ async fn do_run() -> Result<(), CmdError> { ) .required(true), ) + .arg( + arg!( + -g --gz_local_link_addr "gz_local_link_addr" + ) + .required(true) + .value_parser(parse_ipv6), + ) .arg( Arg::new("link_local_links") .short('l') @@ -285,6 +293,7 @@ async fn switch_zone_setup( let info: &String = matches.get_one("baseboard_info").unwrap(); let bootstrap_addr: &Ipv6Addr = matches.get_one("bootstrap_addr").unwrap(); let bootstrap_vnic: &String = matches.get_one("bootstrap_vnic").unwrap(); + let gz_local_link_addr: &Ipv6Addr = matches.get_one("gz_local_link_addr").unwrap(); let links = if let Some(l) = matches.get_many::("link_local_links") { Some(l.collect::>()) @@ -363,6 +372,19 @@ async fn switch_zone_setup( }, )?; + info!( + &log, + "Forwarding bootstrap traffic via {} to {}", + bootstrap_vnic, + gz_local_link_addr + ); + Route::add_bootstrap_route( + BOOTSTRAP_PREFIX, + *gz_local_link_addr, + &bootstrap_vnic, + ) + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + Ok(()) } From 207162b5af3ec5058af9e05541891ae48be48704 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 27 Jun 2024 12:42:17 +1200 Subject: [PATCH 52/56] Clean up --- sled-agent/src/services.rs | 74 ++------------------------------ zone-setup/src/bin/zone-setup.rs | 3 +- 2 files changed, 5 insertions(+), 72 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index b887d2b67f..a20d83995a 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -57,7 +57,6 @@ use illumos_utils::running_zone::{ use illumos_utils::zfs::ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT; use illumos_utils::zone::AddressRequest; use illumos_utils::zpool::ZpoolName; -// use illumos_utils::ExecutionError; use illumos_utils::{execute, PFEXEC}; use internal_dns::resolver::Resolver; use itertools::Itertools; @@ -84,8 +83,7 @@ use omicron_common::api::internal::shared::{ HostPortConfig, RackNetworkConfig, }; use omicron_common::backoff::{ - retry_notify, retry_policy_internal_service_aggressive, //retry_policy_local, - BackoffError, + retry_notify, retry_policy_internal_service_aggressive, BackoffError, }; use omicron_common::ledger::{self, Ledger, Ledgerable}; use omicron_ddm_admin_client::{Client as DdmAdminClient, DdmError}; @@ -94,7 +92,6 @@ use rand::prelude::SliceRandom; use sled_hardware::is_gimlet; use sled_hardware::underlay; use sled_hardware::SledMode; -//use sled_hardware_types::underlay::BOOTSTRAP_PREFIX; use sled_hardware_types::Baseboard; use sled_storage::config::MountConfig; use sled_storage::dataset::{ @@ -2372,8 +2369,7 @@ impl ServiceManager { let mut uplink_service = ServiceBuilder::new("oxide/uplink"); let mut switch_zone_setup_config = - PropertyGroupBuilder::new("config") - .add_property( + PropertyGroupBuilder::new("config").add_property( "gz_local_link_addr", "astring", &format!( @@ -3028,73 +3024,9 @@ impl ServiceManager { .map_err(|err| { Error::io("Failed to setup Switch zone profile", err) })?; + return Ok(RunningZone::boot(installed_zone).await?); } } - - let running_zone = RunningZone::boot(installed_zone).await?; - - // if let Some((bootstrap_name, _)) = bootstrap_name_and_address.as_ref() { - // // Forwarding bootstrap traffic requires the bootstrap interface to exist. - // // We need to wait for it. - // retry_notify( - // retry_policy_local(), - // || async { - // info!( - // self.inner.log, - // "Forwarding bootstrap traffic via {} to {}", - // bootstrap_name, - // self.inner.global_zone_bootstrap_link_local_address, - // ); - // running_zone - // .add_bootstrap_route( - // BOOTSTRAP_PREFIX, - // self.inner.global_zone_bootstrap_link_local_address, - // bootstrap_name, - // ) - // .map_err(|err| { - // match err.err { - // ExecutionError::CommandFailure(ref e) => { - // if e.stderr.contains("no such interface") { - // BackoffError::transient( - // Error::ZoneCommand{ - // intent: "add bootstrap network route".to_string(), - // err, - // } - // ) - // } else { - // BackoffError::permanent( - // Error::ZoneCommand{ - // intent: "add bootstrap network route".to_string(), - // err, - // } - // ) - // } - // } - // _ => { - // BackoffError::permanent( - // Error::ZoneCommand{ - // intent: "add bootstrap network route".to_string(), - // err, - // } - // ) - // } - // } - // }) - // }, - // |err, delay| { - // info!( - // &self.inner.log, - // "Cannot forward bootstrap traffic via {} to {} yet (retrying in {:?})", - // bootstrap_name, - // self.inner.global_zone_bootstrap_link_local_address, - // delay; - // "error" => ?err - // ); - // }, - // ) - // .await?; - // } - Ok(running_zone) } // Ensures that a single Omicron zone is running. diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 6dd911c3c1..bb52a81499 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -293,7 +293,8 @@ async fn switch_zone_setup( let info: &String = matches.get_one("baseboard_info").unwrap(); let bootstrap_addr: &Ipv6Addr = matches.get_one("bootstrap_addr").unwrap(); let bootstrap_vnic: &String = matches.get_one("bootstrap_vnic").unwrap(); - let gz_local_link_addr: &Ipv6Addr = matches.get_one("gz_local_link_addr").unwrap(); + let gz_local_link_addr: &Ipv6Addr = + matches.get_one("gz_local_link_addr").unwrap(); let links = if let Some(l) = matches.get_many::("link_local_links") { Some(l.collect::>()) From 36c95caaed1a2735bdba29fa382bf2e0fd25c276 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 27 Jun 2024 14:09:47 +1200 Subject: [PATCH 53/56] Clean up --gateway flag --- sled-agent/src/services.rs | 20 +++--- smf/zone-network-setup/manifest.xml | 2 +- zone-setup/src/bin/zone-setup.rs | 106 +++++++++++++++------------- 3 files changed, 67 insertions(+), 61 deletions(-) diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index a20d83995a..423d3a1160 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -1424,17 +1424,19 @@ impl ServiceManager { ) -> Result { let datalink = zone.get_control_vnic_name(); + let mut config_builder = PropertyGroupBuilder::new("config"); + config_builder = + config_builder.add_property("datalink", "astring", datalink); + // The switch zone is the only zone that will sometimes have an // unknown underlay address at zone boot on the first scrimlet. - let gateway = match gw_addr { - Some(addr) => addr.to_string(), - None => "unknown".to_string(), - }; - - let mut config_builder = PropertyGroupBuilder::new("config"); - config_builder = config_builder - .add_property("datalink", "astring", datalink) - .add_property("gateway", "astring", gateway); + if let Some(gateway) = gw_addr { + config_builder = config_builder.add_property( + "gateway", + "astring", + gateway.to_string(), + ); + } for s in static_addrs { config_builder = config_builder.add_property( diff --git a/smf/zone-network-setup/manifest.xml b/smf/zone-network-setup/manifest.xml index 8955f5adc0..722618eb47 100644 --- a/smf/zone-network-setup/manifest.xml +++ b/smf/zone-network-setup/manifest.xml @@ -27,7 +27,7 @@ - + diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index bb52a81499..4c68aa41f2 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -24,7 +24,6 @@ use std::io::Write; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::os::unix::fs::chown; use std::path::Path; -use std::str::FromStr; use uzers::{get_group_by_name, get_user_by_name}; use zone_setup::switch_zone_user::SwitchZoneUser; @@ -60,16 +59,6 @@ fn parse_ipv6(s: &str) -> anyhow::Result { s.parse().map_err(|_| anyhow!("ERROR: Invalid IPv6 address")) } -fn parse_gateway(s: &str) -> anyhow::Result> { - if s == "unknown" || s == "" { - return Ok(None); - }; - - let gw = Ipv6Addr::from_str(s) - .map_err(|_| anyhow!("ERROR: Invalid IPv6 address"))?; - Ok(Some(gw)) -} - fn parse_datalink(s: &str) -> anyhow::Result { if s == "unknown" { return Err(anyhow!("ERROR: Missing data link")); @@ -147,11 +136,16 @@ async fn do_run() -> Result<(), CmdError> { .value_parser(parse_datalink), ) .arg( - arg!( - -g --gateway "gateway" - ) - .required(true) - .value_parser(parse_gateway), + // We are taking a list of values so that clap + // allows us to set a flag without values in the + // SMF manifest. This will happen with the switch + // zone when the underlay isn't up yet. + Arg::new("gateway") + .short('g') + .long("gateway") + .num_args(0..=1) + .value_parser(value_parser!(Ipv6Addr)) + .help("Underlay address") ) .arg( Arg::new("static_addrs") @@ -658,7 +652,11 @@ async fn common_nw_set_up( .get_many::("static_addrs") .unwrap() .collect::>(); - let gateway: Option = *matches.get_one("gateway").unwrap(); + let gateway = if let Some(g) = matches.get_many::("gateway") { + Some(g.collect::>()) + } else { + None + }; let zonename = zone::current().await.map_err(|err| { CmdError::Failure(anyhow!( "Could not determine local zone name: {}", @@ -705,45 +703,51 @@ async fn common_nw_set_up( // Only the switch zone will sometimes have an unknown underlay address at zone boot. None => info!(&log, "Underlay is not available yet. Not ensuring there is a default route"), Some(gw) => { - // Ensuring default route with gateway must happen after peer agents have been initialized. - // Omicron zones will be able ensure a default route with gateway immediately, but the - // switch zone on the secondary scrimlet might need a few tries while it waits. - retry_notify( - retry_policy_local(), - || async { - info!(&log, "Ensuring there is a default route"; "gateway" => ?gw); - Route::ensure_default_route_with_gateway(Gateway::Ipv6(gw)) - .map_err(|err| { - match err { - ExecutionError::CommandFailure(ref e) => { - if e.stdout.contains("Network is unreachable") { - BackoffError::transient( - CmdError::Failure(anyhow!(err)), - ) - } else { + if gw.is_empty() { + info!(&log, "Underlay is not available yet. Not ensuring there is a default route"); + } else { + // We can safely retrieve the first address only as the CLI only accepts a single item. + let gw = gw.first().unwrap(); + // Ensuring default route with gateway must happen after peer agents have been initialized. + // Omicron zones will be able ensure a default route with gateway immediately, but the + // switch zone on the secondary scrimlet might need a few tries while it waits. + retry_notify( + retry_policy_local(), + || async { + info!(&log, "Ensuring there is a default route"; "gateway" => ?gw); + Route::ensure_default_route_with_gateway(Gateway::Ipv6(**gw)) + .map_err(|err| { + match err { + ExecutionError::CommandFailure(ref e) => { + if e.stdout.contains("Network is unreachable") { + BackoffError::transient( + CmdError::Failure(anyhow!(err)), + ) + } else { + BackoffError::permanent( + CmdError::Failure(anyhow!(err)), + ) + } + } + _ => { BackoffError::permanent( CmdError::Failure(anyhow!(err)), ) } } - _ => { - BackoffError::permanent( - CmdError::Failure(anyhow!(err)), - ) - } - } - }) - }, - |err, delay| { - info!( - &log, - "Cannot ensure there is a default route yet (retrying in {:?})", - delay; - "error" => ?err - ); - }, - ) - .await?; + }) + }, + |err, delay| { + info!( + &log, + "Cannot ensure there is a default route yet (retrying in {:?})", + delay; + "error" => ?err + ); + }, + ) + .await?; + } } } From fae95de1b5dab3e21e6c4c569f5b31f3108bd015 Mon Sep 17 00:00:00 2001 From: karencfv Date: Thu, 27 Jun 2024 16:49:32 +1200 Subject: [PATCH 54/56] noop commit --- zone-setup/src/bin/zone-setup.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zone-setup/src/bin/zone-setup.rs b/zone-setup/src/bin/zone-setup.rs index 4c68aa41f2..3239129d3c 100644 --- a/zone-setup/src/bin/zone-setup.rs +++ b/zone-setup/src/bin/zone-setup.rs @@ -708,6 +708,7 @@ async fn common_nw_set_up( } else { // We can safely retrieve the first address only as the CLI only accepts a single item. let gw = gw.first().unwrap(); + // Ensuring default route with gateway must happen after peer agents have been initialized. // Omicron zones will be able ensure a default route with gateway immediately, but the // switch zone on the secondary scrimlet might need a few tries while it waits. From 2f7b31378420a1e98a841f878b4db0b87e4c8eb8 Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 15 Jul 2024 18:31:32 +1200 Subject: [PATCH 55/56] Update Dendrite hashes --- package-manifest.toml | 12 ++++++------ tools/dendrite_openapi_version | 2 +- tools/dendrite_stub_checksums | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-manifest.toml b/package-manifest.toml index 66cc6e08aa..ad7d9a00d7 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -643,8 +643,8 @@ only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "dendrite" # TODO: Set to images from main branch. Testing out with images from my PR for now. -source.commit = "e058ad924378bdd7e27a169dda299867472a0503" -source.sha256 = "16798a450f33800e004f915442f6b603f547a6bf16312c26faf087853aec6c4a" +source.commit = "84831440e2fe026ce6705608b5393cd0baef4337" +source.sha256 = "c4fb4920e5bfcff5588ed75966c8f889e7e1e2749ced65cc43d27239a172bef4" output.type = "zone" output.intermediate_only = true @@ -669,8 +669,8 @@ only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "dendrite" # TODO: Set to images from main branch. Testing out with images from my PR for now. -source.commit = "e058ad924378bdd7e27a169dda299867472a0503" -source.sha256 = "8545ddd844653c8ba917ba0eaabf6ec9effc140ea1e16bd1f5f2838c083729d5" +source.commit = "84831440e2fe026ce6705608b5393cd0baef4337" +source.sha256 = "29071f9c16905b500d2b3947966a2ab71c788dac5883ef60de1061d675f60161" output.type = "zone" output.intermediate_only = true @@ -688,8 +688,8 @@ only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "dendrite" # TODO: Set to images from main branch. Testing out with images from my PR for now. -source.commit = "e058ad924378bdd7e27a169dda299867472a0503" -source.sha256 = "d9c1460eb3de99d3303f56b014278eef804cfd3c1bef585cd9b516f6670997f7" +source.commit = "84831440e2fe026ce6705608b5393cd0baef4337" +source.sha256 = "679f6c80ea43f59d5b3339f95441cfacb2fa8d66d0d76e2fffe6186e1a8b0163" output.type = "zone" output.intermediate_only = true diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index f2d6620268..4a97b82447 100755 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="e058ad924378bdd7e27a169dda299867472a0503" +COMMIT="84831440e2fe026ce6705608b5393cd0baef4337" SHA2="12dc61e7c62b2e1ee1cf3c2bf7cdda6bee6ec96925d2fc1c021c6c1a8fdd56cd" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index b9decd7b4b..f0bad02610 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="16798a450f33800e004f915442f6b603f547a6bf16312c26faf087853aec6c4a" -CIDL_SHA256_LINUX_DPD="77de3dd6190120f587a03d4d1cdb54114815829b827f0151167bc5b260836651" +CIDL_SHA256_ILLUMOS="c4fb4920e5bfcff5588ed75966c8f889e7e1e2749ced65cc43d27239a172bef4" +CIDL_SHA256_LINUX_DPD="ede81f858986766bc923580849c592c94323ff0cf9cfd6fd4eb4e72b4b9e9fd3" CIDL_SHA256_LINUX_SWADM="a1308303fd0d8f8ac272288e801beb913f695dcf820dd53f5c03871e6b8674f7" From 4a269b3bb52908a7a75e7e50fa6998b12283cc0b Mon Sep 17 00:00:00 2001 From: karencfv Date: Mon, 22 Jul 2024 11:57:59 +1200 Subject: [PATCH 56/56] update to latest dendrite commit --- package-manifest.toml | 15 ++++++--------- tools/dendrite_openapi_version | 2 +- tools/dendrite_stub_checksums | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/package-manifest.toml b/package-manifest.toml index c54d147f4a..098e15d3b8 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -644,9 +644,8 @@ only_for_targets.image = "standard" # the other `source.*` keys. source.type = "prebuilt" source.repo = "dendrite" -# TODO: Set to images from main branch. Testing out with images from my PR for now. -source.commit = "84831440e2fe026ce6705608b5393cd0baef4337" -source.sha256 = "c4fb4920e5bfcff5588ed75966c8f889e7e1e2749ced65cc43d27239a172bef4" +source.commit = "fb571dc6512b24a777c5a9b2927a50501f6be297" +source.sha256 = "c7971efca6500cee8edf2696ec6b38014af82bacfe88a0e583bb9bb3a591bc8d" output.type = "zone" output.intermediate_only = true @@ -672,9 +671,8 @@ only_for_targets.image = "standard" # the other `source.*` keys. source.type = "prebuilt" source.repo = "dendrite" -# TODO: Set to images from main branch. Testing out with images from my PR for now. -source.commit = "84831440e2fe026ce6705608b5393cd0baef4337" -source.sha256 = "29071f9c16905b500d2b3947966a2ab71c788dac5883ef60de1061d675f60161" +source.commit = "fb571dc6512b24a777c5a9b2927a50501f6be297" +source.sha256 = "0a96670ce203bce7bed6a0e40842d319c2b4b8ee1a2e9210d3713423f8bd00b1" output.type = "zone" output.intermediate_only = true @@ -693,9 +691,8 @@ only_for_targets.image = "standard" # the other `source.*` keys. source.type = "prebuilt" source.repo = "dendrite" -# TODO: Set to images from main branch. Testing out with images from my PR for now. -source.commit = "84831440e2fe026ce6705608b5393cd0baef4337" -source.sha256 = "679f6c80ea43f59d5b3339f95441cfacb2fa8d66d0d76e2fffe6186e1a8b0163" +source.commit = "fb571dc6512b24a777c5a9b2927a50501f6be297" +source.sha256 = "a5bda6b899bff23fccd4dd74224fd1bc44703741054b50552921efa7470cb11a" output.type = "zone" output.intermediate_only = true diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index 4a97b82447..06d02a2a8c 100755 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="84831440e2fe026ce6705608b5393cd0baef4337" +COMMIT="fb571dc6512b24a777c5a9b2927a50501f6be297" SHA2="12dc61e7c62b2e1ee1cf3c2bf7cdda6bee6ec96925d2fc1c021c6c1a8fdd56cd" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index f0bad02610..8b51b0c244 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="c4fb4920e5bfcff5588ed75966c8f889e7e1e2749ced65cc43d27239a172bef4" -CIDL_SHA256_LINUX_DPD="ede81f858986766bc923580849c592c94323ff0cf9cfd6fd4eb4e72b4b9e9fd3" +CIDL_SHA256_ILLUMOS="c7971efca6500cee8edf2696ec6b38014af82bacfe88a0e583bb9bb3a591bc8d" +CIDL_SHA256_LINUX_DPD="8091e83e0f8cfc6627be28ae129e4bd897288106d5dc54b6abfd3b974a22f07c" CIDL_SHA256_LINUX_SWADM="a1308303fd0d8f8ac272288e801beb913f695dcf820dd53f5c03871e6b8674f7"