diff --git a/package-manifest.toml b/package-manifest.toml index b8ffb2756aa..e9be9c6e7c4 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -473,8 +473,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966" -source.sha256 = "0808f331741e02d55e199847579dfd01f3658b21c7122cef8c3f9279f43dbab0" +source.commit = "5632a2faabedc2941b05797a84528915d6af6ef3" +source.sha256 = "f1fc8a84b0eb2432a4d424821a33826eebd546b23a793d6a54cad9a7e1933bff" output.type = "zone" output.intermediate_only = true @@ -498,8 +498,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966" -source.sha256 = "c359de1be5073a484d86d4c58e8656a36002ce1dc38506f97b730e21615ccae1" +source.commit = "5632a2faabedc2941b05797a84528915d6af6ef3" +source.sha256 = "6983438f6702911c85708f8aff1a3bc3ae4878e56151ce63a338a7f81f4ae1e5" output.type = "zone" output.intermediate_only = true @@ -516,8 +516,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966" -source.sha256 = "110bfbfb2cf3d3471f3e3a64d26129c7a02f6c5857f9623ebb99690728c3b2ff" +source.commit = "5632a2faabedc2941b05797a84528915d6af6ef3" +source.sha256 = "e77d424dd5c13834086d3d5875fa9eaccfc0e57ff8b5178b701a6b4000ae8fcb" output.type = "zone" output.intermediate_only = true diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index ff9644773a6..3889be5eff2 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -119,5 +119,6 @@ machine-non-gimlet = [] switch-asic = [] switch-stub = [] switch-softnpu = [] +switch-hypersoftnpu = [] rack-topology-single-sled = [] rack-topology-multi-sled = [] diff --git a/sled-agent/src/bootstrap/pre_server.rs b/sled-agent/src/bootstrap/pre_server.rs index 0c19c30865d..05493f5aa3a 100644 --- a/sled-agent/src/bootstrap/pre_server.rs +++ b/sled-agent/src/bootstrap/pre_server.rs @@ -14,6 +14,7 @@ use super::maghemite; use super::secret_retriever::LrtqOrHardcodedSecretRetriever; use super::server::StartError; use crate::config::Config; +use crate::config::SidecarRevision; use crate::services::ServiceManager; use crate::sled_agent::SledAgent; use crate::storage_manager::StorageManager; @@ -339,6 +340,7 @@ async fn cleanup_all_old_global_state(log: &Logger) -> Result<(), StartError> { } fn enable_mg_ddm(config: &Config, log: &Logger) -> Result<(), StartError> { + info!(log, "finding links {:?}", config.data_links); let mg_addr_objs = underlay::find_nics(&config.data_links) .map_err(StartError::FindMaghemiteAddrObjs)?; if mg_addr_objs.is_empty() { @@ -423,7 +425,16 @@ fn sled_mode_from_config(config: &Config) -> Result { } else if cfg!(feature = "switch-stub") { DendriteAsic::TofinoStub } else if cfg!(feature = "switch-softnpu") { - DendriteAsic::SoftNpu + match config.sidecar_revision { + SidecarRevision::SoftZone(_) => DendriteAsic::SoftNpuZone, + SidecarRevision::SoftPropolis(_) => { + DendriteAsic::SoftNpuPropolisDevice + } + _ => return Err(StartError::IncorrectBuildPackaging( + "sled-agent configured to run on softnpu zone but dosen't \ + have a softnpu sidecar revision", + )), + } } else { return Err(StartError::IncorrectBuildPackaging( "sled-agent configured to run on scrimlet but wasn't \ diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 2473c14566a..fad4b2e94b0 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -28,7 +28,8 @@ pub enum SledMode { #[serde(rename_all = "snake_case")] pub enum SidecarRevision { Physical(String), - Soft(SoftPortConfig), + SoftZone(SoftPortConfig), + SoftPropolis(SoftPortConfig), } #[derive(Debug, Clone, Deserialize)] diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 5fda3c1ae68..cd84c9acd4a 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -347,6 +347,7 @@ pub enum ServiceType { #[serde(skip)] Tfport { pkt_source: String, + asic: DendriteAsic, }, #[serde(skip)] Uplink, diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index a9be0e7c4a2..638fb675d22 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -676,6 +676,11 @@ impl ServiceManager { device: "tofino".to_string(), }); } + ServiceType::Dendrite { + asic: DendriteAsic::SoftNpuPropolisDevice, + } => { + devices.push("/dev/tty03".into()); + } _ => (), } } @@ -741,7 +746,7 @@ impl ServiceManager { for svc in &req.services { match &svc.details { - ServiceType::Tfport { pkt_source } => { + ServiceType::Tfport { pkt_source, asic: _ } => { // The tfport service requires a MAC device to/from which sidecar // packets may be multiplexed. If the link isn't present, don't // bother trying to start the zone. @@ -772,9 +777,13 @@ impl ServiceManager { } Err(_) => { - return Err(Error::MissingDevice { - device: link.to_string(), - }); + if let SidecarRevision::SoftZone(_) = + self.inner.sidecar_revision + { + return Err(Error::MissingDevice { + device: link.to_string(), + }); + } } } } @@ -1815,14 +1824,21 @@ impl ServiceManager { "config/port_config", "/opt/oxide/dendrite/misc/model_config.toml", )?, - DendriteAsic::SoftNpu => { - smfh.setprop("config/mgmt", "uds")?; - smfh.setprop( - "config/uds_path", - "/opt/softnpu/stuff", - )?; + asic @ (DendriteAsic::SoftNpuZone + | DendriteAsic::SoftNpuPropolisDevice) => { + if asic == &DendriteAsic::SoftNpuZone { + smfh.setprop("config/mgmt", "uds")?; + smfh.setprop( + "config/uds_path", + "/opt/softnpu/stuff", + )?; + } + if asic == &DendriteAsic::SoftNpuPropolisDevice { + smfh.setprop("config/mgmt", "uart")?; + } let s = match self.inner.sidecar_revision { - SidecarRevision::Soft(ref s) => s, + SidecarRevision::SoftZone(ref s) => s, + SidecarRevision::SoftPropolis(ref s) => s, _ => { return Err(Error::SidecarRevision( anyhow::anyhow!( @@ -1847,7 +1863,7 @@ impl ServiceManager { }; smfh.refresh()?; } - ServiceType::Tfport { pkt_source } => { + ServiceType::Tfport { pkt_source, asic } => { info!(self.inner.log, "Setting up tfport service"); let is_gimlet = is_gimlet().map_err(|e| { @@ -1882,6 +1898,12 @@ impl ServiceManager { } smfh.setprop("config/pkt_source", pkt_source)?; } + if asic == &DendriteAsic::SoftNpuZone { + smfh.setprop("config/flags", "--sync-only")?; + } + if asic == &DendriteAsic::SoftNpuPropolisDevice { + smfh.setprop("config/pkt_source", pkt_source)?; + } smfh.setprop( "config/host", &format!("[{}]", Ipv6Addr::LOCALHOST), @@ -2509,19 +2531,42 @@ impl ServiceManager { vec![ ServiceType::Dendrite { asic: DendriteAsic::TofinoAsic }, ServiceType::ManagementGatewayService, - ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, + ServiceType::Tfport { + pkt_source: "tfpkt0".to_string(), + asic: DendriteAsic::TofinoAsic, + }, + ServiceType::Uplink, + ServiceType::Wicketd { baseboard }, + ServiceType::Mgd, + ServiceType::MgDdm { mode: "transit".to_string() }, + ] + } + + SledMode::Scrimlet { + asic: asic @ DendriteAsic::SoftNpuPropolisDevice, + } => { + data_links = vec!["vioif0".to_owned()]; + vec![ + ServiceType::Dendrite { asic }, + ServiceType::ManagementGatewayService, ServiceType::Uplink, ServiceType::Wicketd { baseboard }, ServiceType::Mgd, ServiceType::MgDdm { mode: "transit".to_string() }, + ServiceType::Tfport { + pkt_source: "vioif0".to_string(), + asic, + }, + ServiceType::SpSim, ] } // Sled is a scrimlet but is not running the real tofino driver. SledMode::Scrimlet { - asic: asic @ (DendriteAsic::TofinoStub | DendriteAsic::SoftNpu), + asic: + asic @ (DendriteAsic::TofinoStub | DendriteAsic::SoftNpuZone), } => { - if let DendriteAsic::SoftNpu = asic { + if let DendriteAsic::SoftNpuZone = asic { let softnpu_filesystem = zone::Fs { ty: "lofs".to_string(), dir: "/opt/softnpu/stuff".to_string(), @@ -2538,7 +2583,10 @@ impl ServiceManager { ServiceType::Wicketd { baseboard }, ServiceType::Mgd, ServiceType::MgDdm { mode: "transit".to_string() }, - ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, + ServiceType::Tfport { + pkt_source: "tfpkt0".to_string(), + asic, + }, ServiceType::SpSim, ] } diff --git a/sled-hardware/src/illumos/mod.rs b/sled-hardware/src/illumos/mod.rs index 0364c98f14f..c0145b75e8d 100644 --- a/sled-hardware/src/illumos/mod.rs +++ b/sled-hardware/src/illumos/mod.rs @@ -611,34 +611,37 @@ impl HardwareManager { // receiver will receive a tokio::sync::broadcast::error::RecvError::Lagged // error, indicating they should re-scan the hardware themselves. let (tx, _) = broadcast::channel(1024); - let hw = match sled_mode { - // Treat as a possible scrimlet and setup to scan for real Tofino device. - SledMode::Auto - | SledMode::Scrimlet { asic: DendriteAsic::TofinoAsic } => { - HardwareView::new() - } + let hw = + match sled_mode { + // Treat as a possible scrimlet and setup to scan for real Tofino device. + SledMode::Auto + | SledMode::Scrimlet { asic: DendriteAsic::TofinoAsic } => { + HardwareView::new() + } - // Treat sled as gimlet and ignore any attached Tofino device. - SledMode::Gimlet => HardwareView::new_stub_tofino( - // active= - false, - ), + // Treat sled as gimlet and ignore any attached Tofino device. + SledMode::Gimlet => HardwareView::new_stub_tofino( + // active= + false, + ), - // Treat as scrimlet and use the stub Tofino device. - SledMode::Scrimlet { asic: DendriteAsic::TofinoStub } => { - HardwareView::new_stub_tofino(true) - } + // Treat as scrimlet and use the stub Tofino device. + SledMode::Scrimlet { asic: DendriteAsic::TofinoStub } => { + HardwareView::new_stub_tofino(true) + } - // Treat as scrimlet (w/ SoftNPU) and use the stub Tofino device. - // TODO-correctness: - // I'm not sure whether or not we should be treating softnpu - // as a stub or treating it as a different HardwareView variant, - // so this might change. - SledMode::Scrimlet { asic: DendriteAsic::SoftNpu } => { - HardwareView::new_stub_tofino(true) + // Treat as scrimlet (w/ SoftNPU) and use the stub Tofino device. + // TODO-correctness: + // I'm not sure whether or not we should be treating softnpu + // as a stub or treating it as a different HardwareView variant, + // so this might change. + SledMode::Scrimlet { + asic: + DendriteAsic::SoftNpuZone + | DendriteAsic::SoftNpuPropolisDevice, + } => HardwareView::new_stub_tofino(true), } - } - .map_err(|e| e.to_string())?; + .map_err(|e| e.to_string())?; let inner = Arc::new(Mutex::new(hw)); // Force the device tree to be polled at least once before returning. diff --git a/sled-hardware/src/lib.rs b/sled-hardware/src/lib.rs index c81bcddbfbb..654dfd59d9f 100644 --- a/sled-hardware/src/lib.rs +++ b/sled-hardware/src/lib.rs @@ -44,7 +44,8 @@ pub enum HardwareUpdate { pub enum DendriteAsic { TofinoAsic, TofinoStub, - SoftNpu, + SoftNpuZone, + SoftNpuPropolisDevice, } impl std::fmt::Display for DendriteAsic { @@ -55,7 +56,9 @@ impl std::fmt::Display for DendriteAsic { match self { DendriteAsic::TofinoAsic => "tofino_asic", DendriteAsic::TofinoStub => "tofino_stub", - DendriteAsic::SoftNpu => "soft_npu", + DendriteAsic::SoftNpuZone => "soft_npu_zone", + DendriteAsic::SoftNpuPropolisDevice => + "soft_npu_propolis_device", } ) } diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index b4cb7e6cffc..18a7bd27914 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -12,11 +12,11 @@ sled_mode = "scrimlet" # Identifies the revision of the sidecar that is attached, if one is attached. # TODO: This field should be removed once Gimlets have the ability to auto-detect # this information. -sidecar_revision.soft = { front_port_count = 1, rear_port_count = 1 } +sidecar_revision.soft_propolis = { front_port_count = 2, rear_port_count = 4 } # Setting this to true causes sled-agent to always report that its time is # in-sync, rather than querying its NTP zone. -skip_timesync = false +skip_timesync = true # For testing purposes, A file-backed zpool can be manually created with the # following: @@ -28,36 +28,25 @@ skip_timesync = false # different vdev for each. The `create_virtual_hardware.sh` script does this # for you. zpools = [ - "oxi_a462a7f7-b628-40fe-80ff-4e4189e2d62b", - "oxi_b462a7f7-b628-40fe-80ff-4e4189e2d62b", - "oxp_d462a7f7-b628-40fe-80ff-4e4189e2d62b", - "oxp_e4b4dc87-ab46-49fb-a4b4-d361ae214c03", - "oxp_f4b4dc87-ab46-49fb-a4b4-d361ae214c03", - "oxp_14b4dc87-ab46-49fb-a4b4-d361ae214c03", - "oxp_24b4dc87-ab46-49fb-a4b4-d361ae214c03", - "oxp_cd70d7f6-2354-4bf2-8012-55bf9eaf7930", - "oxp_ceb4461c-cf56-4719-ad3c-14430bfdfb60", - "oxp_31bd71cd-4736-4a12-a387-9b74b050396f", - "oxp_616b26df-e62a-4c68-b506-f4a923d8aaf7", + "oxi_a462a4f4-b628-40fe-80ff-4e4189e2d62b", + "oxi_b462a4f4-b628-40fe-80ff-4e4189e2d62b", + "oxp_d462a4f4-b628-40fe-80ff-4e4189e2d62b", + "oxp_e4b4dc84-ab46-49fb-a4b4-d361ae214c03", + "oxp_f4b4dc84-ab46-49fb-a4b4-d361ae214c03", + "oxp_14b4dc84-ab46-49fb-a4b4-d361ae214c03", + "oxp_24b4dc84-ab46-49fb-a4b4-d361ae214c03", ] # Percentage of usable physical DRAM to use for the VMM reservoir, which # guest memory is pulled from. -vmm_reservoir_percentage = 50 - -# Swap device size for the system. The device is a sparsely allocated zvol on -# the internal zpool of the M.2 that we booted from. -# -# If use of the VMM reservoir is configured, it is likely the system will not -# work without a swap device configured. -swap_device_size_gb = 64 +vmm_reservoir_percentage = 0 # An optional data link from which we extract a MAC address. # This is used as a unique identifier for the bootstrap address. # # If empty, this will be equivalent to the first result from: # $ dladm show-phys -p -o LINK -# data_link = "igb0" +data_link = "vioif1" # On a multi-sled system, transit-mode Maghemite runs in the `oxz_switch` zone # to configure routes between sleds. This runs over the Sidecar's rear ports @@ -65,11 +54,17 @@ swap_device_size_gb = 64 # tfportd will create the necessary links and Maghemite will be configured to # use those. But on non-Gimlet systems, you need to specify physical links to # be passed into the `oxz_switch` zone for this purpose. You can skip this if -# you're deploying a single-sled system and just leave the single tfportrear -# as-is. -switch_zone_maghemite_links = ["tfportrear0_0"] +# you're deploying a single-sled system. +switch_zone_maghemite_links = [ + "tfportrear0_0", + "tfportrear1_0", + "tfportrear2_0", + "tfportrear3_0", +] -data_links = ["net0", "net1"] +# The data links that sled-agent will treat as a real gimlet cxgbe0/cxgbe1 +# links. +data_links = ["vioif1", "vioif2"] [log] level = "info" diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index 908cb752e91..1db40208f7f 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -16,6 +16,7 @@ set -x SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" OMICRON_TOP="$SOURCE_DIR/.." +SOFTNPU_MODE=${SOFTNPU_MODE:-zone}; . "$SOURCE_DIR/virtual_hardware.sh" @@ -83,6 +84,9 @@ in the SoftNPU zone later to add those entries." ensure_run_as_root ensure_zpools -ensure_simulated_links "$PHYSICAL_LINK" -warn_if_no_proxy_arp -ensure_softnpu_zone + +if [[ "$SOFTNPU_MODE" == "zone" ]]; then + ensure_simulated_links "$PHYSICAL_LINK" + warn_if_no_proxy_arp + ensure_softnpu_zone +fi diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index c91d1c2e98a..4a052eaa37f 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="343e3a572cc02efe3f8b68f9affd008623a33966" +COMMIT="5632a2faabedc2941b05797a84528915d6af6ef3" SHA2="544ab42ccc7942d8ece9cdc80cd85d002bcf9d5646a291322bf2f79087ab6df0" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index 8fa98114fb4..6016152524e 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="0808f331741e02d55e199847579dfd01f3658b21c7122cef8c3f9279f43dbab0" -CIDL_SHA256_LINUX_DPD="3e276dd553dd7cdb75c8ad023c2cd29b91485fafb94f27097a745b2b7ef5ecea" +CIDL_SHA256_ILLUMOS="f1fc8a84b0eb2432a4d424821a33826eebd546b23a793d6a54cad9a7e1933bff" +CIDL_SHA256_LINUX_DPD="7461b5facc3869ff4dbd81c8d7302d5b350a9609d5a5b49735612ebaa8fbdc63" CIDL_SHA256_LINUX_SWADM="645faf8a93bcae9814b2f116bccd66a54763332b56220e93b66316c853ce13d2"