progress toward running on propolis-based softnpu

package-manifest.toml

@@ -473,8 +473,8 @@ only_for_targets.image = "standard"
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966"
-source.sha256 = "0808f331741e02d55e199847579dfd01f3658b21c7122cef8c3f9279f43dbab0"
+source.commit = "5632a2faabedc2941b05797a84528915d6af6ef3"
+source.sha256 = "f1fc8a84b0eb2432a4d424821a33826eebd546b23a793d6a54cad9a7e1933bff"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -498,8 +498,8 @@ only_for_targets.image = "standard"
 # 2. Copy the output zone image from dendrite/out to omicron/out
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966"
-source.sha256 = "c359de1be5073a484d86d4c58e8656a36002ce1dc38506f97b730e21615ccae1"
+source.commit = "5632a2faabedc2941b05797a84528915d6af6ef3"
+source.sha256 = "6983438f6702911c85708f8aff1a3bc3ae4878e56151ce63a338a7f81f4ae1e5"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -516,8 +516,8 @@ only_for_targets.image = "standard"
 # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966"
-source.sha256 = "110bfbfb2cf3d3471f3e3a64d26129c7a02f6c5857f9623ebb99690728c3b2ff"
+source.commit = "5632a2faabedc2941b05797a84528915d6af6ef3"
+source.sha256 = "e77d424dd5c13834086d3d5875fa9eaccfc0e57ff8b5178b701a6b4000ae8fcb"
 output.type = "zone"
 output.intermediate_only = true
 

sled-agent/Cargo.toml

@@ -119,5 +119,6 @@ machine-non-gimlet = []
 switch-asic = []
 switch-stub = []
 switch-softnpu = []
+switch-hypersoftnpu = []
 rack-topology-single-sled = []
 rack-topology-multi-sled = []

sled-agent/src/bootstrap/pre_server.rs

@@ -14,6 +14,7 @@ use super::maghemite;
 use super::secret_retriever::LrtqOrHardcodedSecretRetriever;
 use super::server::StartError;
 use crate::config::Config;
+use crate::config::SidecarRevision;
 use crate::services::ServiceManager;
 use crate::sled_agent::SledAgent;
 use crate::storage_manager::StorageManager;
@@ -339,6 +340,7 @@ async fn cleanup_all_old_global_state(log: &Logger) -> Result<(), StartError> {
 }
 
 fn enable_mg_ddm(config: &Config, log: &Logger) -> Result<(), StartError> {
+    info!(log, "finding links {:?}", config.data_links);
     let mg_addr_objs = underlay::find_nics(&config.data_links)
         .map_err(StartError::FindMaghemiteAddrObjs)?;
     if mg_addr_objs.is_empty() {
@@ -423,7 +425,16 @@ fn sled_mode_from_config(config: &Config) -> Result<SledMode, StartError> {
             } else if cfg!(feature = "switch-stub") {
                 DendriteAsic::TofinoStub
             } else if cfg!(feature = "switch-softnpu") {
-                DendriteAsic::SoftNpu
+                match config.sidecar_revision {
+                    SidecarRevision::SoftZone(_) => DendriteAsic::SoftNpuZone,
+                    SidecarRevision::SoftPropolis(_) => {
+                        DendriteAsic::SoftNpuPropolisDevice
+                    }
+                    _ => return Err(StartError::IncorrectBuildPackaging(
+                        "sled-agent configured to run on softnpu zone but dosen't \
+                         have a softnpu sidecar revision",
+                    )),
+                }
             } else {
                 return Err(StartError::IncorrectBuildPackaging(
                     "sled-agent configured to run on scrimlet but wasn't \

sled-agent/src/config.rs

@@ -28,7 +28,8 @@ pub enum SledMode {
 #[serde(rename_all = "snake_case")]
 pub enum SidecarRevision {
     Physical(String),
-    Soft(SoftPortConfig),
+    SoftZone(SoftPortConfig),
+    SoftPropolis(SoftPortConfig),
 }
 
 #[derive(Debug, Clone, Deserialize)]

sled-agent/src/params.rs

@@ -347,6 +347,7 @@ pub enum ServiceType {
     #[serde(skip)]
     Tfport {
         pkt_source: String,
+        asic: DendriteAsic,
     },
     #[serde(skip)]
     Uplink,

sled-agent/src/services.rs

@@ -676,6 +676,11 @@ impl ServiceManager {
                         device: "tofino".to_string(),
                     });
                 }
+                ServiceType::Dendrite {
+                    asic: DendriteAsic::SoftNpuPropolisDevice,
+                } => {
+                    devices.push("/dev/tty03".into());
+                }
                 _ => (),
             }
         }
@@ -741,7 +746,7 @@ impl ServiceManager {
 
         for svc in &req.services {
             match &svc.details {
-                ServiceType::Tfport { pkt_source } => {
+                ServiceType::Tfport { pkt_source, asic: _ } => {
                     // The tfport service requires a MAC device to/from which sidecar
                     // packets may be multiplexed.  If the link isn't present, don't
                     // bother trying to start the zone.
@@ -772,9 +777,13 @@ impl ServiceManager {
                             }
 
                             Err(_) => {
-                                return Err(Error::MissingDevice {
-                                    device: link.to_string(),
-                                });
+                                if let SidecarRevision::SoftZone(_) =
+                                    self.inner.sidecar_revision
+                                {
+                                    return Err(Error::MissingDevice {
+                                        device: link.to_string(),
+                                    });
+                                }
                             }
                         }
                     }
@@ -1815,14 +1824,21 @@ impl ServiceManager {
                             "config/port_config",
                             "/opt/oxide/dendrite/misc/model_config.toml",
                         )?,
-                        DendriteAsic::SoftNpu => {
-                            smfh.setprop("config/mgmt", "uds")?;
-                            smfh.setprop(
-                                "config/uds_path",
-                                "/opt/softnpu/stuff",
-                            )?;
+                        asic @ (DendriteAsic::SoftNpuZone
+                        | DendriteAsic::SoftNpuPropolisDevice) => {
+                            if asic == &DendriteAsic::SoftNpuZone {
+                                smfh.setprop("config/mgmt", "uds")?;
+                                smfh.setprop(
+                                    "config/uds_path",
+                                    "/opt/softnpu/stuff",
+                                )?;
+                            }
+                            if asic == &DendriteAsic::SoftNpuPropolisDevice {
+                                smfh.setprop("config/mgmt", "uart")?;
+                            }
                             let s = match self.inner.sidecar_revision {
-                                SidecarRevision::Soft(ref s) => s,
+                                SidecarRevision::SoftZone(ref s) => s,
+                                SidecarRevision::SoftPropolis(ref s) => s,
                                 _ => {
                                     return Err(Error::SidecarRevision(
                                         anyhow::anyhow!(
@@ -1847,7 +1863,7 @@ impl ServiceManager {
                     };
                     smfh.refresh()?;
                 }
-                ServiceType::Tfport { pkt_source } => {
+                ServiceType::Tfport { pkt_source, asic } => {
                     info!(self.inner.log, "Setting up tfport service");
 
                     let is_gimlet = is_gimlet().map_err(|e| {
@@ -1882,6 +1898,12 @@ impl ServiceManager {
                         }
                         smfh.setprop("config/pkt_source", pkt_source)?;
                     }
+                    if asic == &DendriteAsic::SoftNpuZone {
+                        smfh.setprop("config/flags", "--sync-only")?;
+                    }
+                    if asic == &DendriteAsic::SoftNpuPropolisDevice {
+                        smfh.setprop("config/pkt_source", pkt_source)?;
+                    }
                     smfh.setprop(
                         "config/host",
                         &format!("[{}]", Ipv6Addr::LOCALHOST),
@@ -2509,19 +2531,42 @@ impl ServiceManager {
                 vec![
                     ServiceType::Dendrite { asic: DendriteAsic::TofinoAsic },
                     ServiceType::ManagementGatewayService,
-                    ServiceType::Tfport { pkt_source: "tfpkt0".to_string() },
+                    ServiceType::Tfport {
+                        pkt_source: "tfpkt0".to_string(),
+                        asic: DendriteAsic::TofinoAsic,
+                    },
+                    ServiceType::Uplink,
+                    ServiceType::Wicketd { baseboard },
+                    ServiceType::Mgd,
+                    ServiceType::MgDdm { mode: "transit".to_string() },
+                ]
+            }
+
+            SledMode::Scrimlet {
+                asic: asic @ DendriteAsic::SoftNpuPropolisDevice,
+            } => {
+                data_links = vec!["vioif0".to_owned()];
+                vec![
+                    ServiceType::Dendrite { asic },
+                    ServiceType::ManagementGatewayService,
                     ServiceType::Uplink,
                     ServiceType::Wicketd { baseboard },
                     ServiceType::Mgd,
                     ServiceType::MgDdm { mode: "transit".to_string() },
+                    ServiceType::Tfport {
+                        pkt_source: "vioif0".to_string(),
+                        asic,
+                    },
+                    ServiceType::SpSim,
                 ]
             }
 
             // Sled is a scrimlet but is not running the real tofino driver.
             SledMode::Scrimlet {
-                asic: asic @ (DendriteAsic::TofinoStub | DendriteAsic::SoftNpu),
+                asic:
+                    asic @ (DendriteAsic::TofinoStub | DendriteAsic::SoftNpuZone),
             } => {
-                if let DendriteAsic::SoftNpu = asic {
+                if let DendriteAsic::SoftNpuZone = asic {
                     let softnpu_filesystem = zone::Fs {
                         ty: "lofs".to_string(),
                         dir: "/opt/softnpu/stuff".to_string(),
@@ -2538,7 +2583,10 @@ impl ServiceManager {
                     ServiceType::Wicketd { baseboard },
                     ServiceType::Mgd,
                     ServiceType::MgDdm { mode: "transit".to_string() },
-                    ServiceType::Tfport { pkt_source: "tfpkt0".to_string() },
+                    ServiceType::Tfport {
+                        pkt_source: "tfpkt0".to_string(),
+                        asic,
+                    },
                     ServiceType::SpSim,
                 ]
             }

sled-hardware/src/illumos/mod.rs

@@ -611,34 +611,37 @@ impl HardwareManager {
         // receiver will receive a tokio::sync::broadcast::error::RecvError::Lagged
         // error, indicating they should re-scan the hardware themselves.
         let (tx, _) = broadcast::channel(1024);
-        let hw = match sled_mode {
-            // Treat as a possible scrimlet and setup to scan for real Tofino device.
-            SledMode::Auto
-            | SledMode::Scrimlet { asic: DendriteAsic::TofinoAsic } => {
-                HardwareView::new()
-            }
+        let hw =
+            match sled_mode {
+                // Treat as a possible scrimlet and setup to scan for real Tofino device.
+                SledMode::Auto
+                | SledMode::Scrimlet { asic: DendriteAsic::TofinoAsic } => {
+                    HardwareView::new()
+                }
 
-            // Treat sled as gimlet and ignore any attached Tofino device.
-            SledMode::Gimlet => HardwareView::new_stub_tofino(
-                // active=
-                false,
-            ),
+                // Treat sled as gimlet and ignore any attached Tofino device.
+                SledMode::Gimlet => HardwareView::new_stub_tofino(
+                    // active=
+                    false,
+                ),
 
-            // Treat as scrimlet and use the stub Tofino device.
-            SledMode::Scrimlet { asic: DendriteAsic::TofinoStub } => {
-                HardwareView::new_stub_tofino(true)
-            }
+                // Treat as scrimlet and use the stub Tofino device.
+                SledMode::Scrimlet { asic: DendriteAsic::TofinoStub } => {
+                    HardwareView::new_stub_tofino(true)
+                }
 
-            // Treat as scrimlet (w/ SoftNPU) and use the stub Tofino device.
-            // TODO-correctness:
-            // I'm not sure whether or not we should be treating softnpu
-            // as a stub or treating it as a different HardwareView variant,
-            // so this might change.
-            SledMode::Scrimlet { asic: DendriteAsic::SoftNpu } => {
-                HardwareView::new_stub_tofino(true)
+                // Treat as scrimlet (w/ SoftNPU) and use the stub Tofino device.
+                // TODO-correctness:
+                // I'm not sure whether or not we should be treating softnpu
+                // as a stub or treating it as a different HardwareView variant,
+                // so this might change.
+                SledMode::Scrimlet {
+                    asic:
+                        DendriteAsic::SoftNpuZone
+                        | DendriteAsic::SoftNpuPropolisDevice,
+                } => HardwareView::new_stub_tofino(true),
             }
-        }
-        .map_err(|e| e.to_string())?;
+            .map_err(|e| e.to_string())?;
         let inner = Arc::new(Mutex::new(hw));
 
         // Force the device tree to be polled at least once before returning.

sled-hardware/src/lib.rs

@@ -44,7 +44,8 @@ pub enum HardwareUpdate {
 pub enum DendriteAsic {
     TofinoAsic,
     TofinoStub,
-    SoftNpu,
+    SoftNpuZone,
+    SoftNpuPropolisDevice,
 }
 
 impl std::fmt::Display for DendriteAsic {
@@ -55,7 +56,9 @@ impl std::fmt::Display for DendriteAsic {
             match self {
                 DendriteAsic::TofinoAsic => "tofino_asic",
                 DendriteAsic::TofinoStub => "tofino_stub",
-                DendriteAsic::SoftNpu => "soft_npu",
+                DendriteAsic::SoftNpuZone => "soft_npu_zone",
+                DendriteAsic::SoftNpuPropolisDevice =>
+                    "soft_npu_propolis_device",
             }
         )
     }